From 5fe7ceb3067f045556d9dd7b3ad3065915dd3a11 Mon Sep 17 00:00:00 2001 From: Laurent Fainsin Date: Wed, 7 Sep 2022 14:34:32 +0200 Subject: [PATCH] chore: move notebooks in parant folder Former-commit-id: 56075c7a86305a7a9644dfaacd5bdcb62a863baf [formerly 35967dee0a3c159669b50e2823f36ff25e036f1e] Former-commit-id: b98a086c0ea8970562055cc3ad1a025a77b94b35 --- .pre-commit-config.yaml | 2 +- src/{notebooks => }/comp.ipynb.REMOVED.git-id | 0 .../convert.ipynb.REMOVED.git-id | 0 src/{notebooks => }/extract.ipynb | 0 src/notebooks/predict.ipynb.REMOVED.git-id | 1 - src/notebooks/prerender.ipynb | 99 --------------- src/predict.ipynb.REMOVED.git-id | 1 + src/predict.py | 80 ------------ src/prerender.ipynb | 119 ++++++++++++++++++ wandb.yaml | 2 +- 10 files changed, 122 insertions(+), 182 deletions(-) rename src/{notebooks => }/comp.ipynb.REMOVED.git-id (100%) rename src/{notebooks => }/convert.ipynb.REMOVED.git-id (100%) rename src/{notebooks => }/extract.ipynb (100%) delete mode 100644 src/notebooks/predict.ipynb.REMOVED.git-id delete mode 100644 src/notebooks/prerender.ipynb create mode 100644 src/predict.ipynb.REMOVED.git-id delete mode 100644 src/predict.py create mode 100644 src/prerender.ipynb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a441a3c..0cb7485 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v4.1.0" hooks: - - id: check-added-large-files + # - id: check-added-large-files - id: check-executables-have-shebangs - id: check-merge-conflict - id: check-symlinks diff --git a/src/notebooks/comp.ipynb.REMOVED.git-id b/src/comp.ipynb.REMOVED.git-id similarity index 100% rename from src/notebooks/comp.ipynb.REMOVED.git-id rename to src/comp.ipynb.REMOVED.git-id diff --git a/src/notebooks/convert.ipynb.REMOVED.git-id b/src/convert.ipynb.REMOVED.git-id similarity index 100% rename from src/notebooks/convert.ipynb.REMOVED.git-id rename to src/convert.ipynb.REMOVED.git-id diff --git a/src/notebooks/extract.ipynb b/src/extract.ipynb similarity index 100% rename from src/notebooks/extract.ipynb rename to src/extract.ipynb diff --git a/src/notebooks/predict.ipynb.REMOVED.git-id b/src/notebooks/predict.ipynb.REMOVED.git-id deleted file mode 100644 index 42b7af2..0000000 --- a/src/notebooks/predict.ipynb.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -ecf0b9ce39e210bc605fd3eab9db8b1215c35fda \ No newline at end of file diff --git a/src/notebooks/prerender.ipynb b/src/notebooks/prerender.ipynb deleted file mode 100644 index e5c0607..0000000 --- a/src/notebooks/prerender.ipynb +++ /dev/null @@ -1,99 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "from PIL import Image\n", - "\n", - "import albumentations as A\n", - "import torchvision.transforms as T\n", - "\n", - "import numpy as np\n", - "from utils import RandomPaste\n", - "from data.dataset import SyntheticDataset\n", - "\n", - "from pathlib import Path\n", - "\n", - "from joblib import Parallel, delayed\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "transform = A.Compose(\n", - " [\n", - " A.Resize(512, 512),\n", - " A.Flip(),\n", - " A.ColorJitter(),\n", - " RandomPaste(5, \"/media/disk1/lfainsin/SPHERES/\"),\n", - " A.GaussianBlur(),\n", - " A.ISONoise(),\n", - " ],\n", - ")\n", - "\n", - "dataset = SyntheticDataset(image_dir=\"/media/disk1/lfainsin/BACKGROUND/\", transform=transform)\n", - "transform = T.ToPILImage()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def render(i, image, mask):\n", - "\n", - " image = transform(image)\n", - " mask = transform(mask)\n", - "\n", - " path = f\"/media/disk1/lfainsin/TRAIN_prerender/{i}/\"\n", - " Path(path).mkdir(parents=True, exist_ok=True)\n", - " \n", - " image.save(f\"{path}/image.jpg\")\n", - " mask.save(f\"{path}/MASK.PNG\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "Parallel(n_jobs=-1)(delayed(render)(i, image, mask) for i, (image, mask) in enumerate(dataset))\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.0 ('.venv': poetry)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.0" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "dc80d2c03865715c8671359a6bf138f6c8ae4e26ae025f2543e0980b8db0ed7e" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/predict.ipynb.REMOVED.git-id b/src/predict.ipynb.REMOVED.git-id new file mode 100644 index 0000000..9f02ef5 --- /dev/null +++ b/src/predict.ipynb.REMOVED.git-id @@ -0,0 +1 @@ +c1a69536ef50112fdf8384eed234407b31a1dfdb \ No newline at end of file diff --git a/src/predict.py b/src/predict.py deleted file mode 100644 index 7dd0ffd..0000000 --- a/src/predict.py +++ /dev/null @@ -1,80 +0,0 @@ -import argparse -import logging - -import albumentations as A -import numpy as np -import onnx -import onnxruntime -from albumentations.pytorch import ToTensorV2 -from PIL import Image - - -def get_args(): - parser = argparse.ArgumentParser( - description="Predict masks from input images", - ) - parser.add_argument( - "--model", - "-m", - default="model.pth", - metavar="FILE", - help="Specify the file in which the model is stored", - ) - parser.add_argument( - "--input", - "-i", - metavar="INPUT", - help="Filenames of input images", - required=True, - ) - parser.add_argument( - "--output", - "-o", - metavar="OUTPUT", - help="Filenames of output images", - ) - - return parser.parse_args() - - -def sigmoid(x): - return 1 / (1 + np.exp(-x)) - - -if __name__ == "__main__": - args = get_args() - - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - onnx_model = onnx.load(args.model) - onnx.checker.check_model(onnx_model) - - ort_session = onnxruntime.InferenceSession(args.model) - - def to_numpy(tensor): - return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() - - img = Image.open(args.input).convert("RGB") - - logging.info(f"Preprocessing image {args.input}") - transform = A.Compose( - [ - A.ToFloat(max_value=255), - ToTensorV2(), - ], - ) - aug = transform(image=np.asarray(img)) - img = aug["image"] - - logging.info(f"Predicting image {args.input}") - img = img.unsqueeze(0) - - # compute ONNX Runtime output prediction - ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(img)} - ort_outs = ort_session.run(None, ort_inputs) - - img_out_y = ort_outs[0] - - img_out_y = Image.fromarray(np.uint8((img_out_y[0] * 255.0).clip(0, 255)[0]), mode="L") - - img_out_y.save(args.output) diff --git a/src/prerender.ipynb b/src/prerender.ipynb new file mode 100644 index 0000000..3980740 --- /dev/null +++ b/src/prerender.ipynb @@ -0,0 +1,119 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/laurent_fainsin/unet/.venv/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "ename": "ImportError", + "evalue": "cannot import name 'SyntheticDataset' from 'data.dataset' (/home/laurent_fainsin/unet/src/data/dataset.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/home/laurent_fainsin/unet/src/prerender.ipynb Cell 1\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mnumpy\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mnp\u001b[39;00m\n\u001b[1;32m 7\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mutils\u001b[39;00m \u001b[39mimport\u001b[39;00m RandomPaste\n\u001b[0;32m----> 8\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mdata\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mdataset\u001b[39;00m \u001b[39mimport\u001b[39;00m SyntheticDataset\n\u001b[1;32m 10\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mpathlib\u001b[39;00m \u001b[39mimport\u001b[39;00m Path\n\u001b[1;32m 12\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mjoblib\u001b[39;00m \u001b[39mimport\u001b[39;00m Parallel, delayed\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'SyntheticDataset' from 'data.dataset' (/home/laurent_fainsin/unet/src/data/dataset.py)" + ] + } + ], + "source": [ + "from PIL import Image\n", + "\n", + "import albumentations as A\n", + "import torchvision.transforms as T\n", + "\n", + "import numpy as np\n", + "from utils import RandomPaste\n", + "from data.dataset import SyntheticDataset\n", + "\n", + "from pathlib import Path\n", + "\n", + "from joblib import Parallel, delayed\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "transform = A.Compose(\n", + " [\n", + " A.Resize(512, 512),\n", + " A.Flip(),\n", + " A.ColorJitter(),\n", + " RandomPaste(5, \"/media/disk1/lfainsin/SPHERES/\"),\n", + " A.GaussianBlur(),\n", + " A.ISONoise(),\n", + " ],\n", + ")\n", + "\n", + "dataset = SyntheticDataset(image_dir=\"/media/disk1/lfainsin/BACKGROUND/\", transform=transform)\n", + "transform = T.ToPILImage()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def render(i, image, mask):\n", + "\n", + " image = transform(image)\n", + " mask = transform(mask)\n", + "\n", + " path = f\"/media/disk1/lfainsin/TRAIN_prerender/{i}/\"\n", + " Path(path).mkdir(parents=True, exist_ok=True)\n", + " \n", + " image.save(f\"{path}/image.jpg\")\n", + " mask.save(f\"{path}/MASK.PNG\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "Parallel(n_jobs=-1)(delayed(render)(i, image, mask) for i, (image, mask) in enumerate(dataset))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.10 ('.venv': poetry)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "177da549c0efb5403fabf2463095e744d5ed7be0fe3e2a9b077a1765a2012433" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/wandb.yaml b/wandb.yaml index a6677cd..075cb94 100644 --- a/wandb.yaml +++ b/wandb.yaml @@ -28,7 +28,7 @@ WORKERS: value: 16 EPOCHS: - value: 50 + value: 100 TRAIN_BATCH_SIZE: value: 10 VALID_BATCH_SIZE: