modified scraper

scraper/Dockerfile

@@ -6,7 +6,6 @@ WORKDIR /app
 
 # Copy the requirements file into the container
 COPY scraper/requirements.txt .
-# COPY scraper /app/scraper
 
 # Install any needed packages specified in requirements.txt
 RUN pip install --trusted-host pypi.python.org -r requirements.txt
@@ -16,6 +15,7 @@ COPY scraper .
 COPY database /app/database
 COPY config_util.py .
 COPY config.yml .
+COPY .env .
 
 # Specify the command to run on container start
 CMD ["python", "./main.py"]

scraper/main.py

@@ -50,7 +50,6 @@ def scrape_website():
     for manufacturer in soup.select('div.c_container.allmakes a'):
         manufacturer_name = manufacturer.text.strip()
         manufacturer_url = BASE_URL + manufacturer['href']
-        print("manufacturer_name......", manufacturer_name)
 
         for category_url in fetch_categories(manufacturer_url):
             for model_url in fetch_models(category_url):

scraper/requirements.txt

@@ -1,4 +1,6 @@
 beautifulsoup4==4.10.0
 requests==2.26.0
 pyyaml==6.0.1
+azure-storage-blob==12.8.1
+python-dotenv
 # sqlite3==0.0.1