Improved URL check when exporting

Author: dhondta

src/zotero/VERSION.txt

@@ -1 +1 @@
-1.7.1
+1.7.2

src/zotero/__init__.py

@@ -81,6 +81,15 @@
     'video recording':      ":movie_camera:",
     'webpage':              ":earth_americas:",
 }
+URL_CHECK_HEADERS = {
+    'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    'Accept-Language': "en-US,en;q=0.9",
+    'Accept-Encoding': "gzip, deflate, br",
+    'Connection': "close",
+    'Range': "bytes=0-0",
+    'Upgrade-Insecure-Requests': "1",
+    'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0",
+}
 URL_NO_CHECK = (
     "arxiv.org",
     "dial.uclouvain.be",
@@ -90,6 +99,7 @@
     "inria.hal.science",
     "iopscience.iop.org",
     "link.springer.com",
+    "onlinelibrary.wiley.com",
     "researchportal.rma.ac.be",
     "scholar.dsu.edu",
     "web.archive.org",
@@ -101,12 +111,24 @@
     "www.sciencedirect.com",
     "www.scopus.com",
     "www.semanticscholar.org",
+    "www.softpedia.com",
     "www.usenix.org",
 )
 
 
-_check_url = lambda url, nocheck=URL_NO_CHECK: url if url.split("://")[-1].split("/")[0] not in nocheck and \
-                                                      requests.head(url).status_code >= 400 else None
+def _check_url(url, nocheck=URL_NO_CHECK):
+    try:
+        scheme, domain = url.split(":", 1)
+    except ValueError:
+        scheme, domain = "http", url
+    domain = domain.lstrip("/").split("/")[0].split("@")[-1].split(":")[0]
+    if scheme not in ["http", "https"] or ts.is_iterable(nocheck) and domain in nocheck:
+        return
+    response = requests.get(url, headers=URL_CHECK_HEADERS, allow_redirects=True, stream=True)
+    code = response.status_code
+    response.close()
+    if code >= 400:
+        return url
 
 
 def _lower_title(title):
@@ -610,14 +632,14 @@ def _marks(self, marker, filters=None, sort=None, desc=False, limit=None):
             json.dump(self.marks, f)
 
     @ts.try_or_die(exc=ValueError, trace=False)
-    def count(self, filters=None):
+    def count(self, filters=None, **kw):
         """ Count items while applying filters. """
         _, data = self._items(["title"], filters or [])
         print(len(data))
 
     @ts.try_or_die(exc=ValueError, trace=False)
     def export(self, fields=None, filters=None, sort=None, desc=False, limit=None, line_format=None,
-               output_format="xlsx", check_url=False):
+               output_format="xlsx", check_url=False, url_no_check=None, **kw):
         """ Export the selected fields of items while applying filters to an Excel file. """
         if "{stars}" in (line_format or "") and "rank" not in fields:
             fields.append("rank")
@@ -664,11 +686,13 @@ def export(self, fields=None, filters=None, sort=None, desc=False, limit=None, l
                 for row in data:
                     r = row[i_rank]
                     mr = max(mr, float(r if r != "-" else 0))
-            lines, url_check_executor, results = [], ThreadPoolExecutor(max_workers=10), []
+            lines = []
+            if check_url:
+                url_check_executor, results = ThreadPoolExecutor(max_workers=10), {}
             for row in data:
                 d = {k.lower(): v for k, v in zip(headers, row)}
-                if d['url'] not in ["", "-"]:
-                    results.append(url_check_executor.submit(_check_url, d['url']))
+                if check_url and d['url'] not in ["", "-"]:
+                    results[url_check_executor.submit(_check_url, d['url'], url_no_check or URL_NO_CHECK)] = d['url']
                 if "Title" in headers and "Url" in headers:
                     d['lower_title'] = t = _lower_title(d['title'])
                     d['link'] = d['title'] if d['url'] in ["", "-"] else f"[{d['title']}]({d['url']})"
@@ -687,23 +711,25 @@ def export(self, fields=None, filters=None, sort=None, desc=False, limit=None, l
                     s = " :star2:" if r == mr else " :star:"
                     d['stars'] = "" if r < .35 else s if .35 <= r < .65 else 2*s if .65 <= r < .85 else 3*s
                 lines.append(line_format.format(**d))
-            for r in as_completed(results):
-                try:
-                    url = r.result()
-                    if url is not None:
+            if check_url:
+                for r in as_completed(results):
+                    try:
+                        url = results[r]
+                        if r.result() is None:
+                            continue
                         logger.warning(f"Broken link: {url}")
-                except requests.exceptions.ConnectionError:
-                    logger.error(f"Broken link: {url} (connection aborted))")
-                except requests.exceptions.SSLError:
-                    logger.error(f"Broken link: {url} (SSL certificate check failure))")
-            url_check_executor.shutdown()
+                    except requests.exceptions.ConnectionError:
+                        continue
+                    except requests.exceptions.SSLError:
+                        logger.error(f"Broken link: {url} (SSL certificate check failure))")
+                url_check_executor.shutdown()
             r = Report(List(*lines))
         r.filename = "export"
         logger.debug(f"Creating {output_format.upper()} file...")
         getattr(r, output_format)(save_to_file=True)
 
     @ts.try_or_die(exc=ValueError, trace=False)
-    def list(self, field, filters=None, desc=False, limit=None):
+    def list(self, field, filters=None, desc=False, limit=None, **kw):
         """ List field's values while applying filters. """
         if field == "collections":
             l = [c['data']['name'] for c in self.collections]
@@ -731,12 +757,12 @@ def list(self, field, filters=None, desc=False, limit=None):
         print(ts.BorderlessTable([[ZoteroCLI.header(field)]] + data))
 
     @ts.try_or_die(exc=ValueError, trace=False)
-    def mark(self, marker, filters=None, sort=None, desc=False, limit=None):
+    def mark(self, marker, filters=None, sort=None, desc=False, limit=None, **kw):
         """ Mark the selected items as read/unread. """
         self._marks(marker, filters, sort, desc, limit)
 
     @ts.try_or_die(exc=ValueError, trace=False)
-    def plot(self, name, filters=None):
+    def plot(self, name, filters=None, **kw):
         """ Plot a chart given its slug. """
         if name == "software-in-time":
             data = {}
@@ -752,7 +778,7 @@ def plot(self, name, filters=None):
             raise ValueError
 
     @ts.try_or_die(exc=ValueError, trace=False)
-    def show(self, fields=None, filters=None, sort=None, desc=False, limit=None):
+    def show(self, fields=None, filters=None, sort=None, desc=False, limit=None, **kw):
         """ Show the selected fields of items while applying filters. """
         # ensure the 'key' field is included for filtering the items ; then do not keep it if not selected
         output_key = "key" in fields
@@ -774,7 +800,7 @@ def show(self, fields=None, filters=None, sort=None, desc=False, limit=None):
             print(table)
 
     @ts.try_or_die(exc=ValueError, trace=False)
-    def view(self, name, value, fields=None):
+    def view(self, name, value, fields=None, **kw):
         """ View a single item given a field and its value. """
         headers, data = self._items(fields, [f"{name}:{value}"])
         for h, d in zip(headers, data[0]):

src/zotero/__main__.py

@@ -21,6 +21,7 @@
 """
 __examples__  = [
     "count -f \"collections:biblio\" -f \"rank:>1.0\"",
+    "export title date url itemType -f \"collections:doc\" -s \"title\" -l \"{emoji} {link_lower}\" -o md --check-url",
     "export year title itemType numAuthors numPages zscc references what results comments -f \"collections:biblio\" " \
         "-s date -l \">rank:50\"",
     "list attachments",
@@ -31,11 +32,23 @@
 ]
 
 
+def _domains(url_no_check):
+    r = []
+    for url in (url_no_check or []):
+        if ts.is_file(url):
+            with open(url) as f:
+                for l in f:
+                    r.append(l.strip())
+        else:
+            r.append(url.strip())
+    return r
+
+
 def _set_arg(subparser, arg, msg=None):
     """ Shortcut function to set arguments repeated for multiple subparsers. """
     if arg == "filter":
-        subparser.add_argument("-f", "--filter", action="extend", nargs="*", default=[], note="format: [field]:[regex]",
-                               help=msg or "filter to be applied on field's value")
+        subparser.add_argument("-f", "--filter", action="extend", dest="filters", nargs="*", default=[],
+                               note="format: [field]:[regex]", help=msg or "filter to be applied on field's value")
     elif arg == "limit":
         subparser.add_argument("-l", "--limit", help="limit the number of displayed records", note="format: either a "
                                "number or [field]:[number]\n    '<' and '>' respectively indicates ascending or "
@@ -76,11 +89,13 @@ def main():
     _set_arg(ccount, "filter", "filter to be applied while counting")
     _set_arg(ccount, "query")
     cexpt = sparsers.add_parser("export", help="export items to a file", category="manage")
-    cexpt.add_argument("field", nargs="+", help="field to be shown")
+    cexpt.add_argument("fields", nargs="+", help="field to be shown")
     cexpt.add_argument("-l", "--line-format", help="line's format string for outputting as a list")
     cexpt.add_argument("-o", "--output-format", default="xlsx", help="output format",
                        choices=["csv", "html", "json", "md", "pdf", "rst", "xml", "xlsx", "yaml"])
     cexpt.add_argument("-u", "--check-url", action="store_true", help="check for broken URL's")
+    cexpt.add_argument("--url-no-check", nargs="*", help="either a domain or a file with one domain per line for "
+                                                         "skipping URL check")
     _set_args(cexpt, "filter", "limit", "query", "sort")
     if __GPT:
         cingest = sparsers.add_parser("ingest", help="ingest Zotero documents", category="GPT")
@@ -105,17 +120,17 @@ def main():
         cselect = sparsers.add_parser("select", help="select a GPT model", category="GPT")
         cselect.add_argument("name", default=MODEL_DEFAULT_NAME, choices=MODELS, nargs="?", help="model name")
     cshow = sparsers.add_parser("show", help="show a list of items", category="read")
-    cshow.add_argument("field", nargs="*", help="field to be shown")
+    cshow.add_argument("fields", nargs="*", help="field to be shown")
     _set_args(cshow, "filter", "limit", "query", "sort")
     cview = sparsers.add_parser("view", help="view a single item", category="read")
     cview.add_argument("name", help="field name for selection")
     cview.add_argument("value", help="field value to be selected")
-    cview.add_argument("field", nargs="+", help="field to be shown")
+    cview.add_argument("fields", nargs="+", help="field to be shown")
     initialize()
     args.logger = logger
     if getattr(args, "query", None):
-        if hasattr(args, "field") and args.field == ["-"]:
-            args.field = QUERIES[args.query].get('fields', ["title"])
+        if hasattr(args, "fields") and args.fields == ["-"]:
+            args.fields = QUERIES[args.query].get('fields', ["title"])
         args.filter.extend(QUERIES[args.query].get('filter', []))
         if getattr(args, "limit", None) is None:
             args.limit = QUERIES[args.query].get('limit')
@@ -132,25 +147,10 @@ def main():
             if getattr(args, "reset_items", False) and k not in ["items"] + OBJECTS or k == "marks":
                 continue
             CACHE_PATH.joinpath(k + ".json").remove(False)
-    z = ZoteroCLI(args.id, ["user", "group"][args.group or GROUP_FILE.exists()], args.key, logger)
-    if args.command == "count":
-        z.count(args.filter)
-    elif args.command == "export":
-        z.export(args.field, args.filter, args.sort, args.desc, args.limit, args.line_format, args.output_format,
-                 args.check_url)
-    elif args.command == "install":
-        install()
-    elif args.command == "list":
-        z.list(args.field, args.filter, args.desc, args.limit)
-    elif args.command == "mark":
-        args.filter.append("numPages:>0")
-        z.mark(args.marker, args.filter, args.sort, args.desc, args.limit)
-    elif args.command == "plot":
-        z.plot(args.chart)
-    elif args.command == "show":
-        z.show(args.field, args.filter, args.sort, args.desc, args.limit)
-    elif args.command == "view":
-        z.view(args.name, args.value, args.field)
-    elif args.command != "reset":  # handle commands from gpt.py
-        globals()[args.command](**vars(args))
+    if args.command != "reset":
+        if args.command == "export":
+            args.url_no_check = l = _domains(args.url_no_check)
+            args.check_url |= len(l) > 0
+        getattr(ZoteroCLI(args.id, ["user", "group"][args.group or GROUP_FILE.exists()], args.key, logger),
+                args.command, globals().get(args.command))(**vars(args))