first commit

eltociear · Oct 25, 2023 · 6ca6fc2 · 6ca6fc2
1 parent 70645ca
commit 6ca6fc2
Show file tree

Hide file tree

Showing 93 changed files with 11,029 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,189 @@
+# Initially taken from Github's Python gitignore file
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# tests and logs
+tests/fixtures/cached_*_text.txt
+logs/
+lightning_logs/
+lang_code_data/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# vscode
+.vs
+.vscode
+
+# Pycharm
+.idea
+
+# TF code
+tensorflow_code
+
+# Models
+proc_data
+
+# examples
+runs
+/runs_old
+/wandb
+/examples/runs
+/examples/**/*.args
+/examples/rag/sweep
+
+# data
+/data
+serialization_dir
+
+# emacs
+*.*~
+debug.env
+
+# vim
+.*.swp
+
+#ctags
+tags
+
+# pre-commit
+.pre-commit*
+
+# .lock
+*.lock
+
+# DS_Store (MacOS)
+.DS_Store
+# RL pipelines may produce mp4 outputs
+*.mp4
+
+# dependencies
+/transformers
+
+# ruff
+.ruff_cache
+
+# ckpts
+*.ckpt
+
+outputs/*
+
+NeuS/exp/*
+NeuS/test_scenes/*
+NeuS/mesh2tex/*
+neus_configs
+vast/*
+render_results
+experiments/*
+ckpts/*
+neus/*
diff --git a/1gpu.yaml b/1gpu.yaml
@@ -0,0 +1,15 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: 'NO'
+downcast_bf16: 'no'
+gpu_ids: '0'
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false
diff --git a/8gpu.yaml b/8gpu.yaml
@@ -0,0 +1,15 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false
diff --git a/README.md b/README.md
@@ -1,6 +1,67 @@
 # Wonder3D
 Single Image to 3D using Cross-Domain Diffusion
-## [Paper](https://arxiv.org/abs/2310.15008) | [Project page](https://www.xxlong.site/Wonder3D/) 
 ![](assets/fig_teaser.png)
 
 Wonder3D reconstructs highly-detailed textured meshes from a single-view image in only 2 ∼ 3 minutes. Wonder3D first generates consistent multi-view normal maps with corresponding color images via a cross-domain diffusion model, and then leverages a novel normal fusion method to achieve fast and high-quality reconstruction.
+
+## [Paper](https://arxiv.org/abs/2310.15008) | [Project page](https://www.xxlong.site/Wonder3D/) 
+- [x] Inference code and pretrained models.
+- [] Huggingface demo.
+- [] Training code.
+- [] Rendering code for data prepare.
+
+
+### Preparation for inference
+1. Install packages in `requirements.txt`. 
+```angular2html
+conda create -n wonder3d
+conda activate wonder3d
+pip install -r requirements.txt
+```
+2. Download the [checkpoints](https://connecthkuhk-my.sharepoint.com/:f:/g/personal/xxlong_connect_hku_hk/EgSHPyJAtaJFpV_BjXM3zXwB-UMIrT4v-sQwGgw-coPtIA) and into the root folder.
+
+### Inference
+1. Make sure you have the following models.
+```bash
+Wonder3D
+|-- ckpts
+    |-- unet
+    |-- scheduler.bin
+    ...
+```
+2. Predict foreground mask as the alpha channel. We use [Clipdrop](https://clipdrop.co/remove-background) to segment the foreground object interactively. 
+You may also use rembg to remove the backgrounds.
+```bash
+# !pip install rembg
+import rembg
+result = rembg.remove(result)
+result.show()
+```
+3. Run Wonder3d to produce multiview-consistent normal maps and color images.
+```bash
+accelerate launch --config_file 1gpu.yaml test_mvdiffusion_seq.py \
+            --config mvdiffusion-joint-ortho-6views.yaml
+```
+or 
+```bash
+bash run_test.sh
+```
+You can check the results in the folder ./outputs. 
+4. Mesh Extraction
+```bash
+cd ./instant-nsr-pl
+bash run.sh output_folder_path scene_name
+```
+
+## Citation
+If you find this repository useful in your project, please cite the following work. :)
+```
+@misc{long2023wonder3d,
+      title={Wonder3D: Single Image to 3D using Cross-Domain Diffusion}, 
+      author={Xiaoxiao Long and Yuan-Chen Guo and Cheng Lin and Yuan Liu and Zhiyang Dou and Lingjie Liu and Yuexin Ma and Song-Hai Zhang and Marc Habermann and Christian Theobalt and Wenping Wang},
+      year={2023},
+      eprint={2310.15008},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}
+```
diff --git a/configs/mvdiffusion-joint-ortho-6views.yaml b/configs/mvdiffusion-joint-ortho-6views.yaml
@@ -0,0 +1,42 @@
+pretrained_model_name_or_path: 'lambdalabs/sd-image-variations-diffusers'
+pretrained_unet_path: './ckpts/'
+revision: null
+validation_dataset:
+  root_dir: "./example_images" # the folder path stores testing images
+  num_views: 6
+  bg_color: 'white'
+  img_wh: [256, 256]
+  num_validation_samples: 1000
+  crop_size: 192
+  filepaths: ['owl.png']  # the test image names. leave it empty, test all images in the folder
+
+save_dir: 'outputs/'
+
+pred_type: 'joint'
+seed: 42
+validation_batch_size: 1
+dataloader_num_workers: 64
+
+local_rank: -1
+
+pipe_kwargs:
+  camera_embedding_type: 'e_de_da_sincos'
+  num_views: 6
+
+validation_guidance_scales: [3.0]
+pipe_validation_kwargs:
+  eta: 1.0
+validation_grid_nrow: 6
+
+unet_from_pretrained_kwargs:
+  camera_embedding_type: 'e_de_da_sincos'
+  projection_class_embeddings_input_dim: 10  
+  num_views: 6
+  sample_size: 32
+  zero_init_conv_in: false
+  zero_init_camera_projection: false  
+
+num_views: 6
+camera_embedding_type: 'e_de_da_sincos'
+
+enable_xformers_memory_efficient_attention: true
diff --git a/example_images/14_10_29_489_Tiger_1__1.png b/example_images/14_10_29_489_Tiger_1__1.png
diff --git a/example_images/box.png b/example_images/box.png
diff --git a/example_images/bread.png b/example_images/bread.png
diff --git a/example_images/cat.png b/example_images/cat.png
diff --git a/example_images/cat_head.png b/example_images/cat_head.png
diff --git a/example_images/chili.png b/example_images/chili.png
diff --git a/example_images/duola.png b/example_images/duola.png
diff --git a/example_images/halloween.png b/example_images/halloween.png
diff --git a/example_images/head.png b/example_images/head.png
diff --git a/example_images/kettle.png b/example_images/kettle.png
diff --git a/example_images/kunkun.png b/example_images/kunkun.png
diff --git a/example_images/milk.png b/example_images/milk.png
diff --git a/example_images/owl.png b/example_images/owl.png
diff --git a/example_images/poro.png b/example_images/poro.png
diff --git a/example_images/pumpkin.png b/example_images/pumpkin.png
diff --git a/example_images/skull.png b/example_images/skull.png
diff --git a/example_images/stone.png b/example_images/stone.png
diff --git a/example_images/teapot.png b/example_images/teapot.png
diff --git a/example_images/tiger-head-3d-model-obj-stl.png b/example_images/tiger-head-3d-model-obj-stl.png