update to latest

README.md

@@ -61,6 +61,8 @@ Projects like this usually end up with a number of processing steps, and it's be
 - Note:
     - a special run is required, after we finished with amending the description_cache.json files
 
+----
+
 - Command: ./build_cats.py –cache –timeout  TIMEOUT_SEC
 - Input file: description_cache.json
 - Output file: description_cache.json
@@ -72,24 +74,35 @@ Projects like this usually end up with a number of processing steps, and it's be
 - Purpose: like previous command, just uses the firefox browser via selenium package.
     - Some hosts can’t be scanned by regular scanner (example: cloudlflare and other DDOS protection mechanisms involve several http redirects, where javascript code is run to determine the next step; therefore get html by automating the browser)
 
+----
 
 - Command: build_geoip.py
 - Input file: description_cache.json
 - Output file: description_cache.json
 - Purpose: set geoip_lan attribute based on geo-ip lookup of host name
 
+----
+
+- Command: build_lang.py
+- Input file: ui_text_string.txt
+- Output file: description_cache.json
+- Purpose: detects the language of the description, and sets the language_description attribute (needed for automatic translation, but maybe i would be better off with auto...)
+
+----
 
 - Command: build_translate.py –descr
 - Input file: description_cache.json
 - Output file: description_cache.json
 - Purpose: build auto translation of description into supported set of languages (build translations field in each host entry)
 
+----
 
 - Command: build_translate.py –uitext
 - Input file: ui_text_string.txt
 - Output file: ui_text_translated.json
 - Purpose: build auto translation of each user interface string that appears on the page (not site descriptions)
 
+----
 
 - Command: build_translate.py –translate
 - Input files: 
@@ -114,3 +127,4 @@ Now github actions are disabled after a while, when they don't see any action in
 
 Also I am using expect to automate pushing stuff into the repo by the build script (see build subdirectory in this project); sometimes that's a useful tool to know.
 
+Interesting that modern systems have this tendency to evolve into Rube Goldberg Devices; microservices are there by definition, data processing stuff is also there... All by virtue of being divided into small self-contained but interdependent parts...

build_lang.py

@@ -34,8 +34,6 @@ def run_identify_language():
             entry_obj.language_description = descr
             cache.cache_set(base_url, entry_obj)
             num_set += 1
-
-    if cache.write_description_cache():
-        print(f"*** description cache changed, number of items set: {num_set}")
+            cache.write_description_cache()
 
 run_identify_language()

build_translate.py

@@ -68,7 +68,7 @@ def process(self, from_lang_name, to_lang_name, text):
 #            print('------End--------')
 #
 #        return None
-
+#
 class TranslateHelpText:
     def __init__(self, list_of_target_langs):
         self.list_of_target_langs = list_of_target_langs

globs.py

@@ -6,7 +6,7 @@ class Globals:
     ui_text_strings = "ui_text_strings.txt"
     ui_text_translated = "ui_text_translated.json"
 
-    supported_languages = [ 'en', 'de', 'fr', 'ru', 'es', 'ja', 'zh', 'uk' ]
+    supported_languages = [ 'it', 'en', 'de', 'fr', 'ru', 'es', 'ja', 'zh', 'uk' ]
     failed_lookups = "failed_lookups.txt"
 
     flag_list = "flag_list.txt"