Minor updates for EMBER support (#143)

* fix some imports and upgrade to rich click

* some updates to parsing manifest

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix minimal imports; add minimal import test; add email notifications

* fix secret passing; fix gawk compile; expose api_url to internal helpers

* expose --api-url to CLI

* generalize tests

* add pytest marks; temporarily suppress DANDI in CI; debug relative path

* debugging cross-platform

* debugging cross-platform

* debugging cross-platform

* debugging cross-platform

* properly mark remote tests

* fix passing of api URL

* allow missing zarr subdir

* add changelog

---------

Co-authored-by: CodyCBakerPhD <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

.github/workflows/dailies.yml

@@ -0,0 +1,40 @@
+name: Deploy daily tests
+
+on:
+  schedule:
+    - cron: "0 6 * * *"  # Daily at 2am EST
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  DailyTests:
+    uses: ./.github/workflows/testing.yml
+    secrets:
+      CODECOV_CREDENTIALS: ${{ secrets.CODECOV_CREDENTIALS }}
+
+  NotifyOnFailure:
+    runs-on: ubuntu-latest
+    needs: [ DailyTests ]
+    if: ${{ always() && needs.DailyTests.result == 'failure' }}
+    steps:
+      - uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{ secrets.MAIL_USERNAME }}
+          password: ${{ secrets.MAIL_PASSWORD }}
+          subject: "CI Failure: nwb2bids Daily Tests"
+          to: [email protected]  # Add more with comma separation (no spaces)
+          from: s3-log-extraction
+          body: "The daily test workflow failed, please check status at https://github.com/dandi/s3-log-extraction/actions/workflows/dailies.yml"
+
+#  LiveServices:
+#    uses: ./.github/workflows/testing_live_services.yml
+#    secrets:
+#      IP_HASH_SALT: ${{ secrets.IP_HASH_SALT }}
+#      IPINFO_API_KEY: ${{ secrets.IPINFO_API_KEY }}
+#      CODECOV_CREDENTIALS: ${{ secrets.CODECOV_CREDENTIALS }}

.github/workflows/deploy_daily_tests.yml

@@ -1,10 +1,13 @@
-name: Dev tests
+name: Primary testing workflow
 on:
   workflow_call:
     secrets:
       CODECOV_CREDENTIALS:
         required: true
 
+env:
+  S3_LOG_EXTRACTION_PASSWORD: ${{ secrets.S3_LOG_EXTRACTION_PASSWORD }}
+
 jobs:
 
   run:
@@ -13,8 +16,6 @@ jobs:
     # Requirement settings identified as 'DevTests / ubuntu'
     name: ubuntu
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
 
     steps:
       - uses: actions/checkout@v4
@@ -25,11 +26,18 @@ jobs:
           python-version: "3.13"
 
       - name: Global Setup
-        run: |
-          python -m pip install -U pip
-          pip install pytest-cov
+        run: python -m pip install -U pip
+
+      - name: Minimal local installation
+        run: pip install --no-cache-dir .
+
+      - name: Display installed packages and their sources for debugging
+        run: pip list
 
-      - name: Install local checkout
+      - name: Test basic import
+        run: python -c "import s3_log_extraction"
+
+      - name: Full installation
         run: pip install --no-cache-dir .[all] --group all
 
       - name: Display installed packages and their sources for debugging
@@ -38,17 +46,17 @@ jobs:
       - name: Test basic import
         run: python -c "import s3_log_extraction"
 
-#      - name: Run pytest with coverage and printout coverage for debugging
-#        run: |
-#          pytest tests -vv -rsx --cov=s3_log_extraction --cov-report xml:./coverage.xml
-#          cat ./coverage.xml
-#
-#      - name: Upload coverage to Codecov
-#        uses: codecov/codecov-action@v4
-#        with:
-#          token: ${{ secrets.CODECOV_CREDENTIALS }}
-#          file: ./coverage.xml
-#          flags: unittests
-#          name: codecov-umbrella
-#          fail_ci_if_error: true
-#          verbose: true
+      - name: Run pytest with coverage and printout coverage for debugging
+        run: |
+          pytest tests -m "not dandi and not remote" -vv -rsx --cov=s3_log_extraction --cov-report xml:./coverage.xml
+          cat ./coverage.xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_CREDENTIALS }}
+          file: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/testing.yml

@@ -21,8 +21,6 @@ jobs:
     # Requirement settings identified as 'DevTests / ubuntu'
     name: ubuntu
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
 
     steps:
       - uses: actions/checkout@v4
@@ -36,7 +34,7 @@ jobs:
         run: python -m pip install -U pip
 
       - name: Install local checkout
-        run: pip install --no-cache-dir .[dev]
+        run: pip install --no-cache-dir .[all] --group all
 
       - name: Display installed packages and their sources for debugging
         run: pip list
@@ -55,12 +53,12 @@ jobs:
         name: Ensure caching is working
         run: |
           ls ~
-          ls ~/.dandi_s3_log_parser
-          cat ~/.dandi_s3_log_parser/ip_hash_to_region.yaml
+          ls ~/.s3_log_extraction
+          cat ~/.s3_log_extraction/ip_hash_to_region.yaml
 
       - name: Run pytest with coverage and printout coverage for debugging
         run: |
-          pytest test_live_services -vv -rsx --cov=dandi_s3_log_parser --cov-report xml:./coverage.xml
+          pytest -m "remote" -vv -rsx --cov=s3_log_extraction --cov-report xml:./coverage.xml
           cat ./coverage.xml
 
       - name: Upload coverage to Codecov

.github/workflows/testing_live_services.yml

@@ -4,6 +4,22 @@
 
 
 
+# v1.2.0
+
+## Features
+
+Support for child instances of DANDI has been added by way of an `--api-url` flag on the CLI for `s3logextraction update summaries` and an `api_url` parameter for the corresponding API methods.
+
+## Improvements
+
+Upgraded CLI to use `rich_click` for better formatting.
+
+## Fixes
+
+Moved some exposed imports to local levels to allow successful import of package under minimal installation conditions.
+
+
+
 # v1.1.3
 
 ## Features
@@ -17,7 +33,7 @@ Now tracks 'unassociated' access activity for DANDI summaries, which includes al
 
 Added `s3_log_extraction.dataase.bundle_database` method for creating a hive-partitioned Parquet-based database of the extraction cache for easier sharing.
 
-# Improvements
+## Improvements
 
 Added `bogon` labeling for IP addresses that are not routable on the public internet, such as private IPs and reserved ranges. This improves the update iteration of the `s3logextraction update ip regions`.
 

CHANGELOG.md

@@ -6,7 +6,7 @@
 </p>
 <p>
     <a href="https://pypi.org/project/s3-log-extraction/"><img alt="PyPI latest release version" src="https://badge.fury.io/py/s3-log-extraction.svg?id=py&kill_cache=1"></a>
-    <a href="https://github.com/dandi/s3-log-extraction/blob/main/license.txt"><img alt="License: MIT" src="https://img.shields.io/pypi/l/s3-log-extraction.svg"></a>
+    <a href="https://github.com/dandi/s3-log-extraction/blob/main/LICENSE.txt"><img alt="License: MIT" src="https://img.shields.io/pypi/l/s3-log-extraction.svg"></a>
     <a href="https://doi.org/10.5281/zenodo.17147965"><img src="https://zenodo.org/badge/826995164.svg" alt="DOI"></a>
 </p>
 <p>
@@ -21,6 +21,8 @@ Developed for the [DANDI Archive](https://dandiarchive.org/).
 
 Read more about [S3 logging on AWS](https://web.archive.org/web/20240807191829/https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html).
 
+⚠️ This package currently only supports processing of access data (`GET`-type requests); if you wish to use this package for other types of requests (`PUT`/`DELETE`/`HEAD`, etc.) please reach out by raising an issue. ⚠️
+
 
 
 ## Installation

README.md

@@ -8,37 +8,15 @@ source = "vcs"
 [tool.hatch.build.targets.wheel]
 packages = ["src/s3_log_extraction"]
 
-# This is a temporary hack while waiting for hatch to support PEP 735, Dependency Groups,
-#   in creating environments. See https://github.com/pypa/hatch/pull/1922 for more
-#   details.
-#
-# Execute `hatch env create test` to create the environment
-# Execute `hatch run test:pytest tests` to run tests in the hatch environment
-[tool.hatch.envs.test]
-features = ["all"]
-post-install-commands = ["pip install --group test"]
-
-[tool.hatch.envs.coverage]
-templates = "test"  # Set this environment to inherit from the test environment
-post-install-commands = ["pip install --group coverage"]
-
-[tool.hatch.envs.dev]
-features = ["all"]
-post-install-commands = ["pip install --group dev"]
-
-[tool.hatch.envs.all]
-features = ["all"]
-post-install-commands = ["pip install --group all"]
-
 
 
 [project]
 name = "s3-log-extraction"
-version="1.1.3"
+version="1.2.0"
 authors = [
   { name="Cody Baker", email="[email protected]" },
 ]
-description = "Fast extraction of access summary data for S3 logs."
+description = "Fast extraction of access summary data from S3 logs."
 readme = "README.md"
 keywords = [
     "AWS",
@@ -56,10 +34,11 @@ dependencies = [
     "pandas",
     "tqdm",
     "PyYAML",
-    "click",
+    "rich_click",
     "natsort",
     "psutil",
     "cryptography",
+    "pydantic",
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -155,6 +134,7 @@ ignore = [
     "S101",
     "ICN001",
     "INP001",
+    "F821",  # Only way to do annotation of lazy imports...
 ]
 fixable = ["ALL"]
 
@@ -166,8 +146,12 @@ known-first-party = ["s3_log_extraction"]
 "src/s3_log_extraction/_hidden_top_level_imports.py" = ["F401"]  # Must perform imports here even if not exposed
 
 
-# None of these environment variables need to be 'correct' with respect to Drogon
+# None of these environment variables need to be 'correct' with respect to test suite
 [tool.pytest.ini_options]
+markers = [
+    "dandi: mark a test as requiring remote resources",
+    "remote: mark a test as requiring remote resources",
+]
 env = [
     "IPINFO_API_KEY=a1",
     "OPENCAGE_API_KEY=a1",