Add more Sqllogictest-style tests

habedi · habedi · commit e53553740412 · 2025-11-02T21:01:03.000+01:00
diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@
 [![Docs](https://img.shields.io/badge/docs-read-blue?style=flat&labelColor=282c34&logo=read-the-docs)](https://github.com/CogitatorTech/gaggle/tree/main/docs)
 [![License](https://img.shields.io/badge/license-MIT%2FApache--2.0-007ec6?style=flat&labelColor=282c34&logo=open-source-initiative)](https://github.com/CogitatorTech/gaggle)
 
-Kaggle Datasets for DuckDB
+Access and Query Kaggle Datasets from DuckDB
 
 </div>
 
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -39,7 +39,7 @@ It outlines features to be implemented and their current status.
     * [x] CSV and TSV file reading.
     * [x] Parquet file reading.
     * [x] JSON file reading.
-    * [ ] Excel (XLSX) file reading. (Available when DuckDB is built with the Excel reader; replacement scan routes `.xlsx` to `read_excel`.)
+    * [x] Excel (XLSX) file reading.
 * **Querying Datasets**
     * [x] Replacement scan for `kaggle:` URLs.
     * [ ] Virtual table support for lazy loading.
@@ -66,8 +66,8 @@ It outlines features to be implemented and their current status.
     * [x] Detailed error codes for programmatic error handling.
 * **Resilience**
     * [x] Automatic retry on network failures.
-    * [ ] Graceful degradation when Kaggle API is unavailable.
     * [x] Local-only mode for cached datasets (via `GAGGLE_OFFLINE`).
+    * [ ] Graceful degradation when Kaggle API is unavailable.
 
 ### 6. Documentation and Distribution
 
diff --git a/docs/README.md b/docs/README.md
@@ -16,7 +16,7 @@ The table below includes the information about all SQL functions exposed by Gagg
 | 10 | `gaggle_update_dataset(dataset_path VARCHAR)`                   | `VARCHAR`        | Forces update to latest version (ignores cache). Returns local path to freshly downloaded dataset.                        |
 | 11 | `gaggle_version_info(dataset_path VARCHAR)`                     | `VARCHAR (JSON)` | Returns version info: `cached_version`, `latest_version`, `is_current`, `is_cached`.                                      |
 | 12 | `gaggle_json_each(json VARCHAR)`                                | `VARCHAR`        | Expands a JSON object/array into newline-delimited JSON rows with fields: `key`, `value`, `type`, `path`.                 |
-| 13 | `gaggle_file_paths(dataset_path VARCHAR, filename VARCHAR)`     | `VARCHAR`        | Resolves a specific file's local path inside a downloaded dataset.                                                        |
+| 13 | `gaggle_file_path(dataset_path VARCHAR, filename VARCHAR)`      | `VARCHAR`        | Resolves a specific file's local path inside a downloaded dataset.                                                        |
 
 > [!NOTE]
 > Dataset paths must be in the form `owner/dataset` where `owner` is the username and `dataset` is the dataset name on
@@ -220,6 +220,6 @@ Gaggle is made up of two main components:
     - C-compatible FFI surface
 
 2. **C++ DuckDB Bindings (`gaggle/bindings/`)** that:
-    - Defines the custom SQL functions (for example: `gaggle_ls`, `gaggle_file_paths`, `gaggle_search`)
+    - Defines the custom SQL functions (for example: `gaggle_ls`, `gaggle_file_path`, `gaggle_search`)
     - Integrates with DuckDB’s extension system and replacement scans (`'kaggle:...'`)
     - Marshals values between DuckDB vectors and the Rust FFI
diff --git a/docs/examples/README.md b/docs/examples/README.md
@@ -23,3 +23,7 @@ Each file is self‑contained and can be executed in the DuckDB shell (or via `d
 ```bash
 make examples
 ```
+
+> [!NOTE] 
+> Some operations (like search and download) need network access unless `GAGGLE_OFFLINE=1`.
+> When offline, these will fail fast if data is not cached locally (not downloaded already).
diff --git a/docs/examples/e1_core_functionality.sql b/docs/examples/e1_core_functionality.sql
@@ -26,22 +26,22 @@ limit 5;
 
 -- Section 4: download a dataset
 select '## Download a dataset';
-select gaggle_download('owid/covid-latest-data') as download_path;
+select gaggle_download('uciml/iris') as download_path;
 
 -- Section 5: list files (JSON)
 select '## list files (json)';
 select to_json(
          list(struct_pack(name := name, size := size, path := path))
        ) as files_json
-from gaggle_ls('owid/covid-latest-data');
+from gaggle_ls('uciml/iris');
 
 -- Section 5b: list files (table)
 select '## list files (table)';
-select * from gaggle_ls('owid/covid-latest-data') limit 5;
+select * from gaggle_ls('uciml/iris') limit 5;
 
 -- Section 6: get dataset metadata
 select '## get dataset metadata';
-select gaggle_info('owid/covid-latest-data') as dataset_metadata;
+select gaggle_info('uciml/iris') as dataset_metadata;
 
 -- Section 7: get cache information
 select '## Get cache information';
diff --git a/docs/examples/e2_advanced_features.sql b/docs/examples/e2_advanced_features.sql
@@ -6,11 +6,11 @@ load 'build/release/extension/gaggle/gaggle.duckdb_extension';
 select gaggle_set_credentials('your-username', 'your-api-key') as credentials_set;
 
 -- Get path to specific file
-select gaggle_file_paths('habedi/flickr-8k-dataset-clean', 'flickr8k.parquet') as file_path;
+select gaggle_file_path('habedi/flickr-8k-dataset-clean', 'flickr8k.parquet') as file_path;
 
 -- Use the file path with DuckDB's read_parquet via prepared statement (no subqueries in args)
 prepare rp as select * from read_parquet(?) limit 10;
-execute rp(gaggle_file_paths('habedi/flickr-8k-dataset-clean', 'flickr8k.parquet'));
+execute rp(gaggle_file_path('habedi/flickr-8k-dataset-clean', 'flickr8k.parquet'));
 
 -- Section 2: list and process multiple files
 select '## list and process dataset files (json and table)';
@@ -35,7 +35,7 @@ select gaggle_cache_info() as cache_status;
 
 -- Section 4: purge cache if needed
 select '## Purge cache (optional)';
--- select gaggle_purge_cache() as cache_purged;
+-- select gaggle_clear_cache() as cache_cleared;
 
 -- Section 5: Dataset versioning
 select '## Check dataset versions';
diff --git a/docs/examples/e4_json_utils.sql b/docs/examples/e4_json_utils.sql
@@ -0,0 +1,24 @@
+.echo on
+
+-- Example 4: JSON helper utilities
+-- Shows how to expand JSON using gaggle_json_each
+
+select '## Load extension';
+load 'build/release/extension/gaggle/gaggle.duckdb_extension';
+
+select '## Expand JSON values into newline-delimited JSON rows';
+select gaggle_json_each('{"a":1,"b":[true,{"c":"x"}],"d":null}') as rows;
+
+-- Combine with DuckDB JSON functions
+with x as (
+  select gaggle_json_each('{"a":1,"b":[true,{"c":"x"}],"d":null}') as row
+)
+select
+  json_type(row)   as value_type,
+  json_extract(row, '$') as raw,
+  json_extract_string(row, '$.key') as key,
+  json_extract(row, '$.value') as value
+from x;
+
+.echo off
+
diff --git a/docs/examples/e5_cache_ops.sql b/docs/examples/e5_cache_ops.sql
@@ -0,0 +1,24 @@
+.echo on
+
+-- Example 5: Cache operations & housekeeping
+-- Demonstrates gaggle_version, gaggle_cache_info, gaggle_clear_cache, gaggle_enforce_cache_limit
+
+select '## Load extension';
+load 'build/release/extension/gaggle/gaggle.duckdb_extension';
+
+select '## Extension version';
+select gaggle_version() as version;
+
+select '## Cache info (path, size, limit)';
+select gaggle_cache_info() as cache_info_json;
+
+select '## Clear cache (optional)';
+-- Uncomment to clear the local dataset cache
+-- select gaggle_clear_cache() as cache_cleared;
+
+select '## Enforce cache size limit (LRU eviction)';
+-- This triggers cleanup based on the configured limit; safe to run repeatedly
+select gaggle_enforce_cache_limit() as enforced;
+
+.echo off
+
diff --git a/gaggle/src/config.rs b/gaggle/src/config.rs
@@ -4,7 +4,7 @@ use std::cell::RefCell;
 use std::env;
 use std::path::PathBuf;
 
-const DEFAULT_CACHE_DIR_NAME: &str = "gaggle_cache";
+const DEFAULT_CACHE_DIR_NAME: &str = "gaggle";
 
 pub static CONFIG: Lazy<GaggleConfig> = Lazy::new(GaggleConfig::from_env);
 
diff --git a/test/sql/test_gaggle_errors.test b/test/sql/test_gaggle_errors.test
@@ -0,0 +1,34 @@
+# group: [gaggle]
+
+# Error and argument validation tests for Gaggle
+
+statement ok
+pragma enable_verification
+
+# Load the Gaggle extension
+statement ok
+load 'build/release/extension/gaggle/gaggle.duckdb_extension'
+
+# Wrong arity for gaggle_download should error
+statement error
+select gaggle_download('a','b')
+----
+No function matches the given name
+
+# NULL input returns NULL by default (DuckDB default null-handling)
+query I
+select gaggle_info(NULL) is null
+----
+1
+
+# Invalid dataset path format should error
+statement error
+select gaggle_download('owner/too/many/components')
+----
+Failed
+
+# Replacement scan with invalid kaggle URL should error (no slash)
+statement error
+select * from 'kaggle:invalid' limit 1
+----
+kaggle:invalid
diff --git a/test/sql/test_gaggle_integration.test b/test/sql/test_gaggle_integration.test
@@ -1,7 +1,7 @@
 # group: [gaggle]
 
 # Test suite for Gaggle extension integration scenarios
-# Tests realistic usage patterns and workflows
+# Tests realistic usage patterns and commom workflows
 
 statement ok
 pragma enable_verification
diff --git a/test/sql/test_gaggle_json.test b/test/sql/test_gaggle_json.test
@@ -0,0 +1,33 @@
+# group: [gaggle]
+
+# JSON-related tests for Gaggle
+
+statement ok
+pragma enable_verification
+
+# Load the Gaggle extension
+statement ok
+load 'build/release/extension/gaggle/gaggle.duckdb_extension'
+
+# gaggle_json_each should succeed on empty object and produce some valid JSON line (non-empty string)
+query I
+select length(gaggle_json_each('{}')) >= 0
+----
+1
+
+# gaggle_json_each should include keys from nested structures as newline-delimited JSON
+query I
+select gaggle_json_each('{"a":1,"b":[true,{"c":"x"}]}') like '%"a"%'
+----
+1
+
+query I
+select gaggle_json_each('{"a":1,"b":[true,{"c":"x"}]}') like '%"b"%'
+----
+1
+
+# gaggle_json_each should error on invalid JSON
+statement error
+select gaggle_json_each('{not valid json}')
+----
+Failed to parse JSON
diff --git a/test/sql/test_gaggle_types.test b/test/sql/test_gaggle_types.test
@@ -0,0 +1,36 @@
+# group: [gaggle]
+
+# Type shape and return type tests for Gaggle
+
+statement ok
+pragma enable_verification
+
+# Load the Gaggle extension
+statement ok
+load 'build/release/extension/gaggle/gaggle.duckdb_extension'
+
+# Return types for scalar functions
+query T
+select typeof(gaggle_version())
+----
+VARCHAR
+
+query T
+select typeof(gaggle_clear_cache())
+----
+BOOLEAN
+
+query T
+select typeof(gaggle_cache_info())
+----
+VARCHAR
+
+query T
+select typeof(gaggle_file_path('owner/dataset','file.csv'))
+----
+VARCHAR
+
+# Table function schema for gaggle_ls
+query T
+select * from (select name, size, path from gaggle_ls('owner/dataset') limit 0)
+----
diff --git a/vcpkg.json b/vcpkg.json
@@ -1,7 +1,7 @@
 {
     "name": "gaggle",
     "version": "0.1.0",
-    "description": "Gaggle DuckDB Extension",
+    "description": "Gaggle DuckDB Extension (C++ dependencies)",
     "dependencies": [
         {
             "name": "vcpkg-cmake",
