updates and improved stability

Author: HenriWahl

.gitignore

@@ -129,4 +129,4 @@ dmypy.json
 .pyre/
 
 # local config
-config.yaml
+config.yaml

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.13-slim
 
 RUN apt update
 RUN apt -y upgrade

backend/collect.py

@@ -3,6 +3,7 @@
 from datetime import datetime, \
     timezone
 from json import loads
+from json.decoder import JSONDecodeError
 from threading import Thread
 from time import sleep
 
@@ -13,15 +14,18 @@
 
 from backend.config import config
 from backend.connection import gitlab_session
-from backend.helpers import plural_or_not
+from backend.helpers import plural_or_not, \
+    exit
 
 # for now all requested URLs start with this suffix
 API_SUFFIX = '/api/v4'
 
+
 class ContainerImageCache(dict):
     """
     dict-based class to act as cache
     """
+
     def __init__(self):
         super().__init__()
         # actual container repositories cache
@@ -44,7 +48,6 @@ def update_cache(self, container_images):
         self.container_images.update(container_images)
         self.create_index_by_name()
 
-
     def create_index_by_name(self):
         """
         allow to access container repositories by their name to make them searchable
@@ -58,14 +61,18 @@ class CollectorThread(Thread):
     """
     collects info from Gitlab at certain interval
     """
+
     def __init__(self):
         Thread.__init__(self, name='Collector')
 
     def run(self):
         global container_images_cache
         global update_status
         while True:
-            container_images_cache.update_cache(self.collect_container_images())
+            try:
+                container_images_cache.update_cache(self.collect_container_images())
+            except JSONDecodeError as exception:
+                print(f'Exception in collector thread: {exception}')
             sleep(config.update_interval)
 
     def collect_container_images(self):
@@ -76,7 +83,7 @@ def collect_container_images(self):
         # to speedup development and avoid waiting for Gitlab response dump the latter
         if config.load_data:
             import pickle
-            with open('container_images_dump.pickle','rb') as pickle_file:
+            with open('container_images_dump.pickle', 'rb') as pickle_file:
                 container_images = pickle.load(pickle_file)
         else:
             # dict for storage of container image info
@@ -120,7 +127,7 @@ def collect_container_images(self):
         # to speedup development and avoid waiting for Gitlab response dump the latter
         if config.dump_data:
             import pickle
-            with open('container_images_dump.pickle','wb') as pickle_file:
+            with open('container_images_dump.pickle', 'wb') as pickle_file:
                 pickle.dump(container_images, pickle_file)
 
         # make update progress bar unnecessary
@@ -148,6 +155,13 @@ def collect_projects() -> list:
                 projects_total_pages = int(response.headers.get('x-total-pages'))
                 projects_page += 1
                 projects_list += loads(response.text)
+            elif response.status_code == 401:
+                # when token is unauthorized exit immediately
+                exit(f'status_code: {response.status_code} text: {response.text}')
+            else:
+                print(f'status_code: {response.status_code} text: {response.text}')
+                # try agin after a short nap
+                sleep(20)
 
         for project in projects_list:
             # fix None project description
@@ -236,7 +250,8 @@ def collect_project_container_images_tags_humanize(container_images: dict) -> di
                 # add human readable tag image size
                 tag['total_size_human_readable'] = '{:.2MiB}'.format(DataSize(tag['total_size']))
                 # add human readable tag creation date
-                tag['created_at_human_readable'] =  dateutil_parser.parse(tag['created_at']).strftime('%Y-%m-%d %H:%M:%S')
+                tag['created_at_human_readable'] = dateutil_parser.parse(tag['created_at']).strftime(
+                    '%Y-%m-%d %H:%M:%S')
 
             # get age of a container image
             # relativedelta adds all other units like 'years=0' which makes it better comparable
@@ -338,4 +353,4 @@ class UpdateStatus:
 update_status = UpdateStatus()
 
 # to be imported by others
-container_images_cache = ContainerImageCache()
+container_images_cache = ContainerImageCache()

backend/connection.py

@@ -1,11 +1,15 @@
-from requests import Session
+from httpx import Client
 
 from backend.config import config
 
 # gitlab_session to be used to connect to GitLab
-gitlab_session = Session()
+gitlab_session = Client(follow_redirects=True)
 gitlab_session.headers.update({'PRIVATE-TOKEN': config.api.token})
 
 # disable TLS verification if configured
-if config.api.get('verify') == False:
-    gitlab_session.verify = False
+if config.api.get('verify', True) == False:
+    gitlab_session = Client(follow_redirects=True, verify=False)
+else:
+    gitlab_session = Client(follow_redirects=True)
+
+gitlab_session.headers.update({'PRIVATE-TOKEN': config.api.token})

backend/helpers.py

@@ -1,3 +1,5 @@
+from os import _exit
+
 def plural_or_not(count: int, word: str) -> str:
     """
     decide if a word has to be in plural because its appears as multiple
@@ -8,4 +10,15 @@ def plural_or_not(count: int, word: str) -> str:
     if count > 1:
         return f'{count} {word}s'
     else:
-        return f'{count} {word}'
+        return f'{count} {word}'
+
+
+def exit(message='', code=1):
+    """
+    exit with message and code
+    :param message:
+    :param code:
+    :return:
+    """
+    print(message)
+    _exit(code)

requirements.txt

@@ -1,7 +1,7 @@
 datasize
 flask
+httpx
 markdown
 munch
 pyaml
 python-dateutil
-requests