Add more Sqllogictest-style tests

Author: habedi

CMakeLists.txt

@@ -1,8 +1,10 @@
 cmake_minimum_required(VERSION 3.5)
 
 set(CORROSION_VERBOSE_OUTPUT ON)
-set(CMAKE_CXX_STANDARD 11)
-set(CMAKE_CXX_STANDARD_REQUIRED 1)
+
+# We need C++17 for std::filesystem on all platforms
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 
 # --- Fallback platform detection (only if DuckDB build system did not set these) ---
@@ -222,3 +224,20 @@ if(TARGET ${EXTENSION_NAME})
 else()
   message(STATUS "[gaggle] No C++ extension target built; skipping install directive.")
 endif()
+
+# Link filesystem library on older libstdc++ (GCC < 9 requires -lstdc++fs)
+set(_GAGGLE_FS_LIB "")
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+    set(_GAGGLE_FS_LIB stdc++fs)
+  endif()
+endif()
+
+if(_GAGGLE_FS_LIB)
+  if (TARGET ${EXTENSION_NAME})
+    target_link_libraries(${EXTENSION_NAME} ${_GAGGLE_FS_LIB})
+  endif()
+  if (TARGET ${LOADABLE_EXTENSION_NAME})
+    target_link_libraries(${LOADABLE_EXTENSION_NAME} ${_GAGGLE_FS_LIB})
+  endif()
+endif()

README.md

@@ -12,34 +12,34 @@
 [![Docs](https://img.shields.io/badge/docs-read-blue?style=flat&labelColor=282c34&logo=read-the-docs)](https://github.com/CogitatorTech/gaggle/tree/main/docs)
 [![License](https://img.shields.io/badge/license-MIT%2FApache--2.0-007ec6?style=flat&labelColor=282c34&logo=open-source-initiative)](https://github.com/CogitatorTech/gaggle)
 
-Kaggle Datasets for DuckDB
+Access and Query Kaggle Datasets from DuckDB
 
 </div>
 
 ---
 
 Gaggle is a DuckDB extension that allows you to work with Kaggle datasets directly in SQL queries, as if
 they were DuckDB tables.
-It is written in Rust and uses the Kaggle API to search, download, and manage datasets.
+It is written in Rust and uses the Kaggle API to search, download, and manage the datasets.
 
 Kaggle hosts a large collection of very useful datasets for data science and machine learning.
 Accessing these datasets typically involves manually downloading a dataset (as a ZIP file),
 extracting it, loading the files in the dataset into your data science environment, and managing storage and dataset
 updates, etc.
-This workflow can be become complex, especially when working with multiple datasets or when datasets are updated
+This workflow can quickly become complex, especially when working with multiple datasets or when datasets are updated
 frequently.
 Gaggle tries to help simplify this process by hiding the complexity and letting you work with datasets directly inside
 an analytical database like DuckDB that can handle fast queries.
 In essence, Gaggle makes DuckDB into a SQL-enabled frontend for Kaggle datasets.
 
 ### Features
 
-- Has a simple API to interact with Kaggle datasets from DuckDB
+- Provides a simple API to interact with Kaggle datasets from DuckDB
 - Allows you to search, download, and read datasets from Kaggle
-- Supports datasets that contain CSV, Parquet, JSON, and XLSX files (XLSX requires DuckDB's Excel reader to be available in your DuckDB build)
-- Configurable and has built-in caching support
+- Supports datasets that contain CSV, Parquet, JSON, and XLSX files
+- Configurable and has built-in caching of downloaded datasets
 - Thread-safe, fast, and has a low memory footprint
-- Supports dataset versioning and update checks
+- Supports dataset updates and versioning
 
 See the [ROADMAP.md](ROADMAP.md) for planned features and the [docs](docs) folder for detailed documentation.
 
@@ -101,8 +101,8 @@ select gaggle_set_credentials('your-username', 'your-api-key');
 -- Get extension version
 select gaggle_version();
 
--- List files in the downloaded dataset
--- (Note that if the datasets is not downloaded yet, it will be downloaded and cached first)
+-- List files in the dataset
+-- (Note that if the datasets is not downloaded, it will be downloaded and cached automatically)
 select *
 from gaggle_ls('habedi/flickr-8k-dataset-clean') limit 5;
 
@@ -150,19 +150,6 @@ Check out the [examples](docs/examples) directory for SQL scripts that show how
 
 ---
 
-### Configuration
-
-See [CONFIGURATION.md](docs/CONFIGURATION.md) for full details. Main environment variables:
-
-- `GAGGLE_CACHE_DIR` — cache directory path (default: `~/.cache/gaggle`)
-- `GAGGLE_HTTP_TIMEOUT` — HTTP timeout (in seconds)
-- `GAGGLE_HTTP_RETRY_ATTEMPTS` — retry attempts after the initial try
-- `GAGGLE_HTTP_RETRY_DELAY_MS` — initial backoff delay (in milliseconds)
-- `GAGGLE_HTTP_RETRY_MAX_DELAY_MS` — maximum backoff delay cap (in milliseconds)
-- `GAGGLE_LOG_LEVEL` — structured log level for the Rust core (like `INFO` or `DEBUG`)
-- `GAGGLE_OFFLINE` — disable network; only use cached data (downloads fail fast if not cached)
-- `KAGGLE_USERNAME`, `KAGGLE_KEY` — Kaggle credentials (alternative to the SQL call)
-
 ### Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to make a contribution.

ROADMAP.md

@@ -39,7 +39,7 @@ It outlines features to be implemented and their current status.
     * [x] CSV and TSV file reading.
     * [x] Parquet file reading.
     * [x] JSON file reading.
-    * [ ] Excel (XLSX) file reading. (Available when DuckDB is built with the Excel reader; replacement scan routes `.xlsx` to `read_excel`.)
+    * [x] Excel (XLSX) file reading.
 * **Querying Datasets**
     * [x] Replacement scan for `kaggle:` URLs.
     * [ ] Virtual table support for lazy loading.
@@ -66,8 +66,8 @@ It outlines features to be implemented and their current status.
     * [x] Detailed error codes for programmatic error handling.
 * **Resilience**
     * [x] Automatic retry on network failures.
-    * [ ] Graceful degradation when Kaggle API is unavailable.
     * [x] Local-only mode for cached datasets (via `GAGGLE_OFFLINE`).
+    * [ ] Graceful degradation when Kaggle API is unavailable.
 
 ### 6. Documentation and Distribution
 

docs/CONFIGURATION.md

@@ -10,7 +10,7 @@ Gaggle supports configuration via environment variables to customize its behavio
 
 - **Description**: Directory path for caching downloaded Kaggle datasets
 - **Type**: String (path)
-- **Default**: `$XDG_CACHE_HOME/gaggle_cache` (typically `~/.cache/gaggle_cache`)
+- **Default**: `$XDG_CACHE_HOME/gaggle` (typically `~/.cache/gaggle`)
 - **Example**:
   ```bash
   export GAGGLE_CACHE_DIR="/var/cache/gaggle"
@@ -77,38 +77,34 @@ Gaggle supports configuration via environment variables to customize its behavio
     - **Description**: Number of retry attempts after the initial try
     - **Type**: Integer
     - **Default**: `3`
-- **GAGGLE_HTTP_RETRY_DELAY_MS**
-    - **Description**: Initial backoff delay in milliseconds
-    - **Type**: Integer (ms)
-    - **Default**: `1000`
-- **GAGGLE_HTTP_RETRY_MAX_DELAY_MS**
-    - **Description**: Maximum backoff delay cap in milliseconds
-    - **Type**: Integer (ms)
-    - **Default**: `30000`
+- **GAGGLE_HTTP_RETRY_DELAY**
+    - **Description**: Initial backoff delay in seconds
+    - **Type**: Float or integer (seconds)
+    - **Default**: `1`
+- **GAGGLE_HTTP_RETRY_MAX_DELAY**
+    - **Description**: Maximum backoff delay cap in seconds
+    - **Type**: Float or integer (seconds)
+    - **Default**: `30`
 
-  These controls enable exponential backoff with cap across metadata/search/download requests.
+These controls enable exponential backoff with cap across metadata/search/download requests.
 
 #### Download Coordination
 
 When multiple queries attempt to download the same dataset concurrently, Gaggle coordinates using an in-process lock.
 These settings control the wait behavior when a download is already in progress.
 
-- **GAGGLE_DOWNLOAD_WAIT_TIMEOUT_MS**
-  - **Description**: Maximum time a waiting request will block for a concurrent download to finish
-  - **Type**: Integer (milliseconds)
-  - **Default**: `30000` (30 seconds)
+- **GAGGLE_DOWNLOAD_WAIT_TIMEOUT**
+  - **Description**: Maximum time a waiting request will block (seconds)
+  - **Type**: Float or integer (seconds)
+  - **Default**: `30`
   - **Example**:
     ```bash
-    export GAGGLE_DOWNLOAD_WAIT_TIMEOUT_MS=600000 # 10 minutes
-    ```
-- **GAGGLE_DOWNLOAD_WAIT_POLL_MS**
-  - **Description**: Polling interval while waiting on another download
-  - **Type**: Integer (milliseconds)
-  - **Default**: `100`
-  - **Example**:
-    ```bash
-    export GAGGLE_DOWNLOAD_WAIT_POLL_MS=250
+    export GAGGLE_DOWNLOAD_WAIT_TIMEOUT=600 # 10 minutes
     ```
+- **GAGGLE_DOWNLOAD_WAIT_POLL**
+  - **Description**: Polling interval while waiting (seconds)
+  - **Type**: Float or integer (seconds)
+  - **Default**: `0.1`
 
 #### Logging Configuration
 
@@ -144,9 +140,10 @@ These settings control the wait behavior when a download is already in progress.
   - **Type**: Boolean (`1`, `true`, `yes`, `on` to enable)
   - **Default**: `false`
   - **Effects**:
-    - gaggle_download(...) fails if the dataset isn’t cached.
-    - Version checks use cached `.downloaded` metadata when available; otherwise return "unknown".
-    - Search and metadata calls will still attempt network; consider avoiding them in offline mode.
+    - `gaggle_download(...)` fails if the dataset isn’t cached.
+    - `gaggle_version_info` reports `latest_version` as "unknown" if no cache metadata exists.
+    - `gaggle_is_current` and other version checks use cached `.downloaded` metadata when available.
+    - `gaggle_search` and `gaggle_info` also fail fast in offline mode (no network attempts).
   - **Example**:
     ```bash
     export GAGGLE_OFFLINE=1
@@ -185,9 +182,9 @@ export GAGGLE_CACHE_DIR="/var/lib/gaggle/cache"
 export GAGGLE_CACHE_SIZE_LIMIT_MB=51200     # 50GB
 export GAGGLE_HTTP_TIMEOUT=120              # 2 minutes
 export GAGGLE_HTTP_RETRY_ATTEMPTS=5         # Retry up to 5 times
-export GAGGLE_HTTP_RETRY_DELAY_MS=2000      # 2 second initial delay
-export GAGGLE_HTTP_RETRY_MAX_DELAY_MS=30000 # Cap backoff at 30s
-export GAGGLE_LOG_LEVEL=WARN                # Production logging (planned)
+export GAGGLE_HTTP_RETRY_DELAY=2            # 2 second initial delay
+export GAGGLE_HTTP_RETRY_MAX_DELAY=30       # Cap backoff at 30s
+export GAGGLE_LOG_LEVEL=WARN                # Production logging
 
 ## Set Kaggle credentials
 export KAGGLE_USERNAME="your-username"
@@ -202,10 +199,10 @@ export KAGGLE_KEY="your-api-key"
 ```bash
 ## Development setup with verbose logging
 export GAGGLE_CACHE_DIR="./dev-cache"
-export GAGGLE_LOG_LEVEL=DEBUG               ## Detailed debug logs (planned)
+export GAGGLE_LOG_LEVEL=DEBUG               ## Detailed debug logs
 export GAGGLE_HTTP_TIMEOUT=10               ## Shorter timeout for dev
 export GAGGLE_HTTP_RETRY_ATTEMPTS=1         ## Fail fast in development
-export GAGGLE_HTTP_RETRY_DELAY_MS=250       ## Quick retry
+export GAGGLE_HTTP_RETRY_DELAY=0.25         ## Quick retry (250ms)
 
 ## Run DuckDB
 ./build/release/duckdb
@@ -217,8 +214,8 @@ export GAGGLE_HTTP_RETRY_DELAY_MS=250       ## Quick retry
 ## Configuration for slow or unreliable networks
 export GAGGLE_HTTP_TIMEOUT=300              ## 5 minute timeout
 export GAGGLE_HTTP_RETRY_ATTEMPTS=10        ## Many retries
-export GAGGLE_HTTP_RETRY_DELAY_MS=5000      ## 5 second initial delay
-export GAGGLE_HTTP_RETRY_MAX_DELAY_MS=60000 ## Cap at 60s
+export GAGGLE_HTTP_RETRY_DELAY=5            ## 5 second initial delay
+export GAGGLE_HTTP_RETRY_MAX_DELAY=60       ## Cap at 60s
 
 ./build/release/duckdb
 ```
@@ -230,10 +227,11 @@ export GAGGLE_HTTP_RETRY_MAX_DELAY_MS=60000 ## Cap at 60s
 export GAGGLE_OFFLINE=1
 
 # Attempt to download a dataset (will fail if not cached)
-gaggle download username/dataset-name
+SELECT gaggle_download('username/dataset-name');
 
-# Querying metadata or searching will still attempt network access
-gaggle info username/dataset-name
+# Querying metadata or searching will fail fast in offline mode
+SELECT gaggle_info('username/dataset-name');
+SELECT gaggle_search('keyword', 1, 10);
 ```
 
 ### Configuration Verification
@@ -253,21 +251,27 @@ SELECT gaggle_search('housing', 1, 10);
 
 -- Get dataset metadata
 SELECT gaggle_info('username/dataset-name');
+
+-- Retrieve last error string (or NULL if none)
+SELECT gaggle_last_error();
 ```
 
 ### Retry Policy Details
 
 Gaggle implements retries with exponential backoff for HTTP requests. The number of attempts, initial delay, and
 maximum delay can be tuned with the environment variables above.
 
-### Logging Levels (planned)
+### Logging Levels
 
-Detailed logging control via `GAGGLE_LOG_LEVEL` is planned but not yet implemented.
+Detailed logging control via `GAGGLE_LOG_LEVEL` is implemented.
 
-### Notes
+### Units
 
-- Cache directory and HTTP timeout are checked at runtime. Changing `GAGGLE_CACHE_DIR` or `GAGGLE_HTTP_TIMEOUT` takes
-  effect for subsequent operations in the same process.
-- Kaggle credentials can be provided via environment variables, config file, or the `gaggle_set_credentials()` SQL
-  function.
-- Invalid values fall back to sensible defaults.
+- Storage sizes are reported in megabytes (MB) throughout the API and SQL functions.
+- Timeouts and retry delays are configured in seconds via environment variables with clean names (no unit suffixes). For example: `GAGGLE_HTTP_RETRY_DELAY=1.5`.
+
+```sql
+-- Example cache info (note size is in MB only)
+SELECT gaggle_cache_info();
+-- {"path":"...","size_mb":42,"limit_mb":102400,"usage_percent":0,"is_soft_limit":true,"type":"local"}
+```

docs/ERROR_CODES.md

@@ -159,7 +159,8 @@ Network error occurred during communication with Kaggle API.
 4. **Retry with backoff:**
    ```bash
    export GAGGLE_HTTP_RETRY_ATTEMPTS=5
-   export GAGGLE_HTTP_RETRY_DELAY_MS=2000
+   export GAGGLE_HTTP_RETRY_DELAY=2
+   export GAGGLE_HTTP_RETRY_MAX_DELAY=30
    ```
 
 5. **Check firewall settings:**

docs/README.md

@@ -16,7 +16,7 @@ The table below includes the information about all SQL functions exposed by Gagg
 | 10 | `gaggle_update_dataset(dataset_path VARCHAR)`                   | `VARCHAR`        | Forces update to latest version (ignores cache). Returns local path to freshly downloaded dataset.                        |
 | 11 | `gaggle_version_info(dataset_path VARCHAR)`                     | `VARCHAR (JSON)` | Returns version info: `cached_version`, `latest_version`, `is_current`, `is_cached`.                                      |
 | 12 | `gaggle_json_each(json VARCHAR)`                                | `VARCHAR`        | Expands a JSON object/array into newline-delimited JSON rows with fields: `key`, `value`, `type`, `path`.                 |
-| 13 | `gaggle_file_paths(dataset_path VARCHAR, filename VARCHAR)`     | `VARCHAR`        | Resolves a specific file's local path inside a downloaded dataset.                                                        |
+| 13 | `gaggle_file_path(dataset_path VARCHAR, filename VARCHAR)`      | `VARCHAR`        | Resolves a specific file's local path inside a downloaded dataset.                                                        |
 
 > [!NOTE]
 > Dataset paths must be in the form `owner/dataset` where `owner` is the username and `dataset` is the dataset name on
@@ -188,15 +188,20 @@ See [CONFIGURATION.md](CONFIGURATION.md) for full details. Key environment varia
 - `GAGGLE_CACHE_DIR` — cache directory path (default: `~/.cache/gaggle`)
 - `GAGGLE_HTTP_TIMEOUT` — HTTP timeout (in seconds)
 - `GAGGLE_HTTP_RETRY_ATTEMPTS` — retry attempts after the initial try
-- `GAGGLE_HTTP_RETRY_DELAY_MS` — initial backoff delay (in milliseconds)
-- `GAGGLE_HTTP_RETRY_MAX_DELAY_MS` — maximum backoff delay cap (in milliseconds)
+- `GAGGLE_HTTP_RETRY_DELAY` — initial backoff delay (in seconds)
+- `GAGGLE_HTTP_RETRY_MAX_DELAY` — maximum backoff delay cap (in seconds)
 - `GAGGLE_LOG_LEVEL` — structured log level for the Rust core (e.g., `INFO`, `DEBUG`)
 - `GAGGLE_OFFLINE` — disable network; only use cached data (downloads fail fast if not cached)
-- `KAGGLE_USERNAME`, `KAGGLE_KEY` — Kaggle credentials (alternative to the SQL call)
+- `KAGGLE_USERNAME` and `KAGGLE_KEY` — Kaggle credentials (alternative to the SQL call)
 
 > [!NOTE]
 > Environment variables are case-sensitive on Unix-like systems. Changes take effect for subsequent operations in the same process.
 
+#### Units
+
+- Storage sizes are reported in megabytes (MB) across SQL/API (for example: `gaggle_cache_info()` returns `size_mb`).
+- Timeouts and retry delays are configured in seconds (via clean environment variables without unit suffixes).
+
 ### Replacement Scan Readers
 
 Gaggle selects the DuckDB reader based on file extension:
@@ -220,6 +225,6 @@ Gaggle is made up of two main components:
     - C-compatible FFI surface
 
 2. **C++ DuckDB Bindings (`gaggle/bindings/`)** that:
-    - Defines the custom SQL functions (for example: `gaggle_ls`, `gaggle_file_paths`, `gaggle_search`)
+    - Defines the custom SQL functions (for example: `gaggle_ls`, `gaggle_file_path`, `gaggle_search`)
     - Integrates with DuckDB’s extension system and replacement scans (`'kaggle:...'`)
     - Marshals values between DuckDB vectors and the Rust FFI

docs/examples/README.md

@@ -23,3 +23,7 @@ Each file is self‑contained and can be executed in the DuckDB shell (or via `d
 ```bash
 make examples
 ```
+
+> [!NOTE]
+> Some operations (like search and download) need network access unless `GAGGLE_OFFLINE=1`.
+> When offline, these will fail fast if data is not cached locally (not downloaded already).

docs/examples/e1_core_functionality.sql

@@ -26,22 +26,22 @@ limit 5;
 
 -- Section 4: download a dataset
 select '## Download a dataset';
-select gaggle_download('owid/covid-latest-data') as download_path;
+select gaggle_download('uciml/iris') as download_path;
 
 -- Section 5: list files (JSON)
 select '## list files (json)';
 select to_json(
          list(struct_pack(name := name, size := size, path := path))
        ) as files_json
-from gaggle_ls('owid/covid-latest-data');
+from gaggle_ls('uciml/iris');
 
 -- Section 5b: list files (table)
 select '## list files (table)';
-select * from gaggle_ls('owid/covid-latest-data') limit 5;
+select * from gaggle_ls('uciml/iris') limit 5;
 
 -- Section 6: get dataset metadata
 select '## get dataset metadata';
-select gaggle_info('owid/covid-latest-data') as dataset_metadata;
+select gaggle_info('uciml/iris') as dataset_metadata;
 
 -- Section 7: get cache information
 select '## Get cache information';