add integration tests for all problem types (h2oai#380)

* piplock

* mv app_utils

* fix imports

* add prep data test

* add tests for all three problem types

* Update requirements.txt

* BLEU until h2oai#381 is fixed

* add coverage

* need_gpus decorator

* fix merge issue

* paths

Makefile

@@ -69,7 +69,12 @@ black: pipenv
 
 .PHONY: test
 test: reports
-	@bash -c 'set -o pipefail; export PYTHONPATH=$(PWD); $(PIPENV) run pytest -v -s --junitxml=reports/junit.xml \
+	@bash -c 'set -o pipefail; export PYTHONPATH=$(PWD); \
+	$(PIPENV) run pytest -v --junitxml=reports/junit.xml \
+	--html=./reports/pytest.html \
+	--cov=llm_studio \
+	--cov-report term \
+	--cov-report html:./reports/coverage.html \
     -o log_cli=true -o log_level=INFO -o log_file=reports/tests.log \
     tests/* 2>&1 | tee reports/tests.log'
 

Pipfile

@@ -51,17 +51,18 @@ hf-transfer = "0.1.3"
 peft = "0.4.0"
 
 [dev-packages]
-pytest = "==7.4.0"
 black = "==23.7.0"
 coverage = "==7.2.7"
+flake8 = "==6.1.0"
 flake8-black = "==0.3.6"
 flake8-isort = "==6.0.0"
-flake8 = "==6.1.0"
 isort = "==5.12.0"
+mypy = "==1.4.1"
+pytest = "==7.4.0"
 pytest-cov = "==4.1.0"
-wheel = "==0.41.0"
 pytest-dependency = "==0.5.1"
+pytest-html = "3.2.0"
+types-pyyaml = ">=6.0"
 types-requests = ">=2.31"
 types-toml = ">=0.10"
-types-pyyaml = ">=6.0"
-mypy = "==1.4.1"
+wheel = "==0.41.0"

README.md

@@ -1,5 +1,5 @@
-<p align="center"><img src="app_utils/static/llm-studio-logo-light.png#gh-dark-mode-only"></p>
-<p align="center"><img src="app_utils/static/llm-studio-logo.png#gh-light-mode-only"></p>
+<p align="center"><img src="llm_studio.app_utils/static/llm-studio-logo-light.png#gh-dark-mode-only"></p>
+<p align="center"><img src="llm_studio.app_utils/static/llm-studio-logo.png#gh-light-mode-only"></p>
 
 <h3 align="center">
     <p>Welcome to H2O LLM Studio, a framework and no-code GUI designed for<br />

app.py

@@ -1,16 +1,16 @@
 import logging
 import os
 
-from app_utils.sections.chat import show_chat_is_running_dialog
+from llm_studio.app_utils.sections.chat import show_chat_is_running_dialog
 from llm_studio.src.utils.logging_utils import initialize_logging
 
 os.environ["MKL_THREADING_LAYER"] = "GNU"
 
 from h2o_wave import Q, app, copy_expando, main, ui  # noqa: F401
 
-from app_utils.handlers import handle
-from app_utils.initializers import initialize_app, initialize_client
-from app_utils.sections.common import interface
+from llm_studio.app_utils.handlers import handle
+from llm_studio.app_utils.initializers import initialize_app, initialize_client
+from llm_studio.app_utils.sections.common import interface
 
 logger = logging.getLogger(__name__)
 

documentation/docs/faqs.md

@@ -74,7 +74,7 @@ This will allow you to revert to your existing version if needed.
 
 ### Once I have the [LoRA](guide/experiments/experiment-settings.md#lora), what is the recommended way of utilizing it with the base model?
 
-You can also export the LoRA weights. You may add them to the files to be exported [here](https://github.com/h2oai/h2o-llmstudio/blob/main/app_utils/sections/experiment.py#L1552). Before exporting, the LoRA weights are merged back into the original LLM backbone weights to make downstream tasks easier. You do not need to have PEFT, or anything else for your deployment.
+You can also export the LoRA weights. You may add them to the files to be exported [here](https://github.com/h2oai/h2o-llmstudio/blob/main/llm_studio/app_utils/sections/experiment.py#L1552). Before exporting, the LoRA weights are merged back into the original LLM backbone weights to make downstream tasks easier. You do not need to have PEFT, or anything else for your deployment.
 
 ---
 

app_utils/handlers.py → llm_studio/app_utils/handlers.py

@@ -5,9 +5,9 @@
 import torch
 from h2o_wave import Q
 
-from app_utils.sections.chat import chat_tab, chat_update
-from app_utils.sections.common import delete_dialog
-from app_utils.sections.dataset import (
+from llm_studio.app_utils.sections.chat import chat_tab, chat_update
+from llm_studio.app_utils.sections.common import delete_dialog
+from llm_studio.app_utils.sections.dataset import (
     dataset_delete_current_datasets,
     dataset_delete_single,
     dataset_display,
@@ -19,7 +19,7 @@
     dataset_merge,
     dataset_newexperiment,
 )
-from app_utils.sections.experiment import (
+from llm_studio.app_utils.sections.experiment import (
     experiment_delete,
     experiment_display,
     experiment_download_logs,
@@ -32,18 +32,22 @@
     experiment_start,
     experiment_stop,
 )
-from app_utils.sections.home import home
-from app_utils.sections.project import (
+from llm_studio.app_utils.sections.home import home
+from llm_studio.app_utils.sections.project import (
     current_experiment_compare,
     current_experiment_list_compare,
     current_experiment_list_delete,
     current_experiment_list_stop,
     experiment_rename_action_workflow,
     list_current_experiments,
 )
-from app_utils.sections.settings import settings
-from app_utils.utils import add_model_type, load_user_settings, save_user_settings
-from app_utils.wave_utils import report_error, wave_utils_handle_error
+from llm_studio.app_utils.sections.settings import settings
+from llm_studio.app_utils.utils import (
+    add_model_type,
+    load_user_settings,
+    save_user_settings,
+)
+from llm_studio.app_utils.wave_utils import report_error, wave_utils_handle_error
 
 logger = logging.getLogger(__name__)
 

app_utils/hugging_face_utils.py → llm_studio/app_utils/hugging_face_utils.py

@@ -7,8 +7,8 @@
 import transformers
 from jinja2 import Environment, FileSystemLoader
 
-from app_utils.sections.chat import load_cfg_model_tokenizer
-from app_utils.utils import hf_repo_friendly_name, save_hf_yaml, set_env
+from llm_studio.app_utils.sections.chat import load_cfg_model_tokenizer
+from llm_studio.app_utils.utils import hf_repo_friendly_name, save_hf_yaml, set_env
 from llm_studio.src.utils.modeling_utils import check_disk_space
 
 

app_utils/initializers.py → llm_studio/app_utils/initializers.py

@@ -6,7 +6,7 @@
 from bokeh.resources import Resources as BokehResources
 from h2o_wave import Q
 
-from app_utils.sections.common import interface
+from llm_studio.app_utils.sections.common import interface
 from llm_studio.src.utils.config_utils import load_config_py, save_config_yaml
 
 from .config import default_cfg
@@ -115,7 +115,7 @@ async def initialize_app(q: Q) -> None:
 
     logger.info("Initializing app ...")
 
-    icons_pth = "app_utils/static/"
+    icons_pth = "llm_studio/app_utils/static/"
     (q.app["icon_path"],) = await q.site.upload([f"{icons_pth}/icon.png"])
 
     script_sources = []

app_utils/sections/chat.py → llm_studio/app_utils/sections/chat.py

@@ -14,7 +14,12 @@
 from h2o_wave import ui
 from transformers import AutoTokenizer, TextStreamer
 
-from app_utils.utils import get_experiments, get_ui_elements, parse_ui_elements, set_env
+from llm_studio.app_utils.utils import (
+    get_experiments,
+    get_ui_elements,
+    parse_ui_elements,
+    set_env,
+)
 from llm_studio.src.datasets.text_utils import get_tokenizer
 from llm_studio.src.models.text_causal_language_modeling_model import Model
 from llm_studio.src.utils.config_utils import load_config_yaml

app_utils/sections/common.py → llm_studio/app_utils/sections/common.py

@@ -3,8 +3,8 @@
 
 from h2o_wave import Q, ui
 
-from app_utils.cards import card_zones
-from app_utils.config import default_cfg
+from llm_studio.app_utils.cards import card_zones
+from llm_studio.app_utils.config import default_cfg
 
 logger = logging.getLogger(__name__)
 

app_utils/sections/dataset.py → llm_studio/app_utils/sections/dataset.py

@@ -9,10 +9,10 @@
 from h2o_wave import Q, ui
 from h2o_wave.types import ImageCard, MarkupCard, StatListItem, Tab
 
-from app_utils.config import default_cfg
-from app_utils.db import Dataset
-from app_utils.sections.experiment import experiment_start
-from app_utils.utils import (
+from llm_studio.app_utils.config import default_cfg
+from llm_studio.app_utils.db import Dataset
+from llm_studio.app_utils.sections.experiment import experiment_start
+from llm_studio.app_utils.utils import (
     add_model_type,
     check_valid_upload_content,
     clean_error,
@@ -33,7 +33,7 @@
     s3_download,
     s3_file_options,
 )
-from app_utils.wave_utils import busy_dialog, ui_table_from_df
+from llm_studio.app_utils.wave_utils import busy_dialog, ui_table_from_df
 from llm_studio.src.utils.config_utils import (
     load_config_py,
     load_config_yaml,

app_utils/sections/experiment.py → llm_studio/app_utils/sections/experiment.py

@@ -17,11 +17,14 @@
 from h2o_wave import Q, data, ui
 from sqlitedict import SqliteDict
 
-from app_utils.config import default_cfg
-from app_utils.hugging_face_utils import get_model_card, publish_model_to_hugging_face
-from app_utils.sections.chat import chat_tab, load_cfg_model_tokenizer
-from app_utils.sections.common import clean_dashboard
-from app_utils.utils import (
+from llm_studio.app_utils.config import default_cfg
+from llm_studio.app_utils.hugging_face_utils import (
+    get_model_card,
+    publish_model_to_hugging_face,
+)
+from llm_studio.app_utils.sections.chat import chat_tab, load_cfg_model_tokenizer
+from llm_studio.app_utils.sections.common import clean_dashboard
+from llm_studio.app_utils.utils import (
     add_model_type,
     flatten_dict,
     get_cfg_list_items,
@@ -40,7 +43,7 @@
     set_env,
     start_experiment,
 )
-from app_utils.wave_utils import busy_dialog, ui_table_from_df, wave_theme
+from llm_studio.app_utils.wave_utils import busy_dialog, ui_table_from_df, wave_theme
 from llm_studio.src.datasets.text_utils import get_tokenizer
 from llm_studio.src.tooltips import tooltips
 from llm_studio.src.utils.config_utils import (

app_utils/sections/home.py → llm_studio/app_utils/sections/home.py

@@ -2,15 +2,15 @@
 import torch
 from h2o_wave import Q, data, ui
 
-from app_utils.config import default_cfg
-from app_utils.sections.common import clean_dashboard
-from app_utils.utils import (
+from llm_studio.app_utils.config import default_cfg
+from llm_studio.app_utils.sections.common import clean_dashboard
+from llm_studio.app_utils.utils import (
     get_datasets,
     get_experiments,
     get_gpu_usage,
     get_single_gpu_usage,
 )
-from app_utils.wave_utils import ui_table_from_df, wave_theme
+from llm_studio.app_utils.wave_utils import ui_table_from_df, wave_theme
 from llm_studio.src.utils.export_utils import get_size_str
 
 

app_utils/sections/project.py → llm_studio/app_utils/sections/project.py

@@ -3,14 +3,14 @@
 
 from h2o_wave import Q, ui
 
-from app_utils.sections.experiment import (
+from llm_studio.app_utils.sections.experiment import (
     experiment_compare,
     experiment_list,
     experiment_rename_action,
     experiment_rename_form,
     get_table_and_message_item_indices,
 )
-from app_utils.utils import get_experiments_status
+from llm_studio.app_utils.utils import get_experiments_status
 
 logger = logging.getLogger(__name__)
 

app_utils/sections/settings.py → llm_studio/app_utils/sections/settings.py

@@ -3,7 +3,7 @@
 import torch
 from h2o_wave import Q, ui
 
-from app_utils.sections.common import clean_dashboard
+from llm_studio.app_utils.sections.common import clean_dashboard
 from llm_studio.src.loggers import Loggers
 
 

app_utils/utils.py → llm_studio/app_utils/utils.py

@@ -32,7 +32,7 @@
 from pandas.core.frame import DataFrame
 from sqlitedict import SqliteDict
 
-from app_utils.db import Experiment
+from llm_studio.app_utils.db import Experiment
 from llm_studio.src import possible_values
 from llm_studio.src.utils.config_utils import (
     _get_type_annotation_error,

app_utils/wave_utils.py → llm_studio/app_utils/wave_utils.py

@@ -7,7 +7,7 @@
 from h2o_wave import Q, expando_to_dict, ui
 from h2o_wave.types import Component
 
-from app_utils.sections.common import clean_dashboard
+from llm_studio.app_utils.sections.common import clean_dashboard
 
 from .config import default_cfg
 

publish_to_hugging_face.py

@@ -3,8 +3,8 @@
 import os
 import sys
 
-from app_utils import hugging_face_utils
-from app_utils.utils import hf_repo_friendly_name
+from llm_studio.app_utils import hugging_face_utils
+from llm_studio.app_utils.utils import hf_repo_friendly_name
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="")