From 703b34ba04652ed63c08c85397f4a58d9b6d428e Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:21:42 -0400 Subject: [PATCH 01/10] added opencode tool --- .mise.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.mise.toml b/.mise.toml index 3fcfcae..036a2c2 100644 --- a/.mise.toml +++ b/.mise.toml @@ -5,3 +5,4 @@ _.python.venv = { path = ".venv", create = false } python = "3.10" poetry = { version = "2.3", pyproject = "pyproject.toml" } pre-commit = "3" +opencode = "latest" From 4d0df5e8fcce6ebb169befb2905a59f2435632c9 Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:22:27 -0400 Subject: [PATCH 02/10] wiring up agent design context --- AGENTS.md | 71 +++++ notes/00-overview.md | 47 +++ notes/01-models.md | 147 +++++++++ notes/02-migrations.md | 89 ++++++ notes/03-cli-tools.md | 77 +++++ notes/04-database-connection.md | 99 ++++++ notes/05-development.md | 182 +++++++++++ notes/06-authorization-design.md | 530 +++++++++++++++++++++++++++++++ notes/README.md | 23 ++ 9 files changed, 1265 insertions(+) create mode 100644 AGENTS.md create mode 100644 notes/00-overview.md create mode 100644 notes/01-models.md create mode 100644 notes/02-migrations.md create mode 100644 notes/03-cli-tools.md create mode 100644 notes/04-database-connection.md create mode 100644 notes/05-development.md create mode 100644 notes/06-authorization-design.md create mode 100644 notes/README.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6aa14e2 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,71 @@ +# Agent Instructions for cape-cod-db + +## MANDATORY: Project Context Protocol + +**Context is NOT optional. All work on this project MUST include context maintenance.** + +### Loading Protocol (REQUIRED) + +At the start of EVERY session, you MUST: +1. Read ALL files in `notes/*.md` to load current project context +2. Verify you understand the project structure, architecture, and current state +3. If notes/ doesn't exist or is incomplete, update it before proceeding with work + +### Maintenance Protocol (CRITICAL) + +When making ANY changes to code: +- Update relevant context notes IN THE SAME SESSION as code changes +- Add new notes files if new subsystems/concepts are introduced +- Keep notes synchronized with code - outdated context is worse than no context + +### Completion Checklist + +Before marking work as complete, verify: +- [ ] All relevant notes/*.md files have been read +- [ ] Context notes reflect current code state +- [ ] New functionality is documented in appropriate notes +- [ ] Changes to architecture/patterns are reflected in notes +- [ ] Test fixtures (fixtures/test/test_data.sql and cleanup_test_data.sql) are synchronized with schema changes +- [ ] Linting and type checking pass: `poetry run ruff check cape_cod_db/ && poetry run pyright` + +**Work is incomplete without synchronized context. No exceptions.** + +## Project-Specific Guidelines + +### Development Commands + +- **Linting**: `poetry run ruff check cape_cod_db/` (check linting) or `poetry run ruff format cape_cod_db/` (auto-format) +- **Type checking**: `poetry run pyright` (configured via pyrightconfig.json) +- **Code formatting**: `poetry run black cape_cod_db/` (Black) or `poetry run isort cape_cod_db/` (import sorting) +- **Testing**: TBD (check if test framework exists before assuming) +- **Database Connection**: Requires `DB_URL` environment variable (set in `.env` file) +- **Test Data**: Load/cleanup test fixtures: + ```bash + source .env && psql "$DB_URL" -f fixtures/test/test_data.sql + source .env && psql "$DB_URL" -f fixtures/test/cleanup_test_data.sql + ``` +- **Migration Commands**: + ```bash + # Load DB_URL from .env and run capedb commands + source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" current --verbose + source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" upgrade head + source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" downgrade -1 + + # Or use alembic directly + alembic -c cape_cod_db/alembic.ini revision --autogenerate -m "description" + ``` + +### Code Conventions + +- Python 3.10+ with SQLModel ORM +- Black formatting (line length 80) - migrations excluded +- Use existing patterns from models.py for new models +- All tables inherit from CapeModel base class +- Database migrations via Alembic (never bypass migrations) +- CLI scripts defined in pyproject.toml [tool.poetry.scripts] + +### Security + +- Never commit database URLs, passwords, or credentials +- Never log PII/PHI (see User model __repr__ TODO) +- Dev-only PostgreSQL setup is documented in README diff --git a/notes/00-overview.md b/notes/00-overview.md new file mode 100644 index 0000000..83a312c --- /dev/null +++ b/notes/00-overview.md @@ -0,0 +1,47 @@ +# Project Overview + +## Purpose + +`cape-cod-db` is a Python library providing SQLModel definitions for the CAPE environment database. It serves as the canonical source of database schema definitions and migrations for the CAPE system. + +## Technology Stack + +- **Python**: 3.10+ +- **ORM**: SQLModel (combines SQLAlchemy and Pydantic) +- **Database**: PostgreSQL 18.x (primary target) +- **Migrations**: Alembic +- **Build System**: Poetry +- **Formatting**: Black (line length 80) +- **Linting**: Ruff +- **Type Checking**: Pyright + +## Package Structure + +``` +cape_cod_db/ +├── __init__.py # Empty module init +├── models.py # All SQLModel table definitions +├── database.py # Database engine and connection setup +├── alembic.ini # Alembic configuration (can be overridden) +├── migrations/ # Alembic migration system +│ ├── env.py # Alembic environment config +│ ├── script.py.mako # Template for new migrations +│ └── versions/ # Individual migration files +└── cli/ # Command-line tools + ├── capedb.py # Primary migration tool (wraps Alembic) + └── app.py # Direct table creation (non-migration) +``` + +## Key Characteristics + +- **Library Package**: Installed as dependency in other CAPE projects +- **Single Models File**: All table definitions in `models.py` (may split if it becomes painful) +- **Base Class Pattern**: All tables inherit from `CapeModel` +- **Migration-First**: Alembic is the primary way to manage schema changes +- **Security Focus**: No PII/PHI in logs, no credentials in code + +## Distribution + +- Packaged via Poetry +- Version: 0.2.0 +- Exports: `capedb` and `capedb-app` CLI scripts diff --git a/notes/01-models.md b/notes/01-models.md new file mode 100644 index 0000000..30c4207 --- /dev/null +++ b/notes/01-models.md @@ -0,0 +1,147 @@ +# Database Models + +## Architecture + +All database models are defined in `cape_cod_db/models.py` using SQLModel. + +## Base Class: CapeModel + +All CAPE database tables inherit from `CapeModel`, which provides: + +```python +class CapeModel(SQLModel): + created_at: datetime # Auto-set on creation (UTC) + last_edited: datetime # Auto-updated on modification (UTC) +``` + +**Key Features:** +- Automatic timestamp tracking +- UTC timezone enforcement +- `onupdate` hook for `last_edited` (ORM only, not raw SQL) + +**Important:** When manipulating records via raw SQL, `created_at` and `last_edited` are NOT automatically handled. + +## Current Models + +### User + +**Location:** `models.py:26-47` + +**Fields:** +- `id: int | None` - Primary key (auto-increment) +- `first_name: str` - Required +- `last_name: str` - Required +- `email: str` - Required, unique, indexed +- `created_at: datetime` - Inherited from CapeModel +- `last_edited: datetime` - Inherited from CapeModel + +**TODOs:** +1. Add email validation (need to select validator library) +2. Fix `__repr__` to exclude PII/PHI for safe logging + +**Table Name:** `user` (lowercase, SQLModel default) + +### Tributary + +**Location:** `models.py:50-66` + +**Fields:** +- `id: int | None` - Primary key (auto-increment) +- `name: str` - Required, unique, indexed (logical compartment name) +- `description: str | None` - Optional description +- `created_at: datetime` - Inherited from CapeModel +- `last_edited: datetime` - Inherited from CapeModel + +**Purpose:** Represents a logical data compartment with associated AWS resources. Tributaries group users and resources together for authorization. + +**Table Name:** `tributary` + +### UserTributary + +**Location:** `models.py:69-93` + +**Fields:** +- `id: int | None` - Primary key (auto-increment) +- `user_id: int` - Foreign key to User (CASCADE on delete) +- `tributary_id: int` - Foreign key to Tributary (CASCADE on delete) +- `granted_by: int | None` - Foreign key to User (nullable, SET NULL on delete) +- `granted_at: datetime | None` - Timestamp when access was granted +- `created_at: datetime` - Inherited from CapeModel +- `last_edited: datetime` - Inherited from CapeModel + +**Constraints:** +- Unique constraint on (user_id, tributary_id) - user can only be in tributary once +- Index on user_id for fast user → tributary lookups + +**Purpose:** Many-to-many junction table between Users and Tributaries, with audit trail. + +**Table Name:** `usertributary` + +### Resource + +**Location:** `models.py:96-122` + +**Fields:** +- `id: int | None` - Primary key (auto-increment) +- `tributary_id: int` - Foreign key to Tributary (no CASCADE - explicit control) +- `resource_type: str` - Type of resource (e.g., "s3_bucket", "ec2_instance") +- `resource_identifier: str` - Platform-specific identifier (e.g., S3 path, ARN) +- `access_pattern: str` - Access mode (e.g., "read", "write", "connect") +- `metadata: dict | None` - JSONB field for flexible additional data (indexed with GIN) +- `created_at: datetime` - Inherited from CapeModel +- `last_edited: datetime` - Inherited from CapeModel + +**Constraints:** +- Unique constraint on (tributary_id, resource_type, resource_identifier, access_pattern) +- Index on tributary_id for fast tributary → resource lookups +- GIN index on metadata JSONB field for efficient queries + +**Purpose:** Platform-agnostic resource definitions linked to tributaries. Supports AWS, Azure, GCP, etc. + +**Standard Pattern:** Each tributary typically has 4 S3 resources: raw_uploads (write), clean_uploads (read), raw_results (write), clean_results (read). + +**Table Name:** `resource` + +### UserAttribute + +**Location:** `models.py:125-147` + +**Fields:** +- `id: int | None` - Primary key (auto-increment) +- `user_id: int` - Foreign key to User (CASCADE on delete) +- `attribute_key: str` - Attribute name (e.g., "user_status", "role") +- `attribute_value: str` - Attribute value (e.g., "quarantine", "admin") +- `created_at: datetime` - Inherited from CapeModel +- `last_edited: datetime` - Inherited from CapeModel + +**Constraints:** +- Unique constraint on (user_id, attribute_key) - one value per key per user +- Index on user_id for fast user → attribute lookups + +**Purpose:** Key-value attributes for users to handle non-tributary authorization (quarantine, admin roles, suspension, etc.). + +**Standard Attributes:** +- `user_status`: Values like "active", "quarantine", "suspended", "deactivated" +- `role`: Values like "admin", "analyst", "viewer" + +**Table Name:** `userattribute` + +## Model Conventions + +1. **Inheritance:** All tables inherit from `CapeModel` with `table=True` +2. **Single File:** All models in one module for now +3. **Type Hints:** Use Python 3.10+ union syntax (`int | None`) +4. **Primary Keys:** Use auto-increment integers with `Field(default=None, primary_key=True)` +5. **Indexes:** Add `index=True` to Field() for commonly queried columns +6. **Unique Constraints:** Add `unique=True` to Field() as needed + +## Adding New Models + +When adding new models: +1. Create class inheriting from `CapeModel` with `table=True` +2. Define all fields with proper types and Field() constraints +3. Import the model in `migrations/env.py` (required for Alembic autogenerate) +4. Generate migration: `alembic revision --autogenerate -m "description"` +5. Review generated migration before applying +6. Update this note with new model documentation +7. **Update test data fixtures** in `fixtures/test/test_data.sql` and `fixtures/test/cleanup_test_data.sql` diff --git a/notes/02-migrations.md b/notes/02-migrations.md new file mode 100644 index 0000000..3252f16 --- /dev/null +++ b/notes/02-migrations.md @@ -0,0 +1,89 @@ +# Alembic Migrations + +## Overview + +Database schema changes are managed via Alembic. This is the **mandatory** approach for schema changes in production environments. + +## Migration System Components + +### env.py Configuration + +**Location:** `cape_cod_db/migrations/env.py` + +**Key Features:** +- Custom naming conventions for constraints (ix_, uq_, ck_, fk_, pk_) +- Type change detection enabled (`compare_type=True`) +- Batch mode for SQLite compatibility (`render_as_batch=True`) +- Target metadata from `SQLModel.metadata` + +**Model Registration:** New models MUST be imported in `env.py:10` for autogenerate to work: +```python +from cape_cod_db.models import User # Add new models here +``` + +### Existing Migrations + +**Location:** `cape_cod_db/migrations/versions/` + +1. `eecb735a6c3b_create_user_table.py` - Initial User table creation +2. `6001985fea71_add_email_to_user_table.py` - Added email field to User + +## capedb CLI Tool + +**Location:** `cape_cod_db/cli/capedb.py` + +**Purpose:** Wrapper around Alembic providing simplified migration commands. + +**Available Commands:** +- `capedb upgrade ` - Upgrade to specified revision (use `head` for latest) +- `capedb downgrade ` - Downgrade to specified revision +- `capedb current [--verbose]` - Show current database version + +**Configuration Options (in priority order):** +1. Command-line: `capedb -c ` or via `ALEMBIC_CONFIG` env var +2. Auto-discovery: Looks for `alembic.ini` in current directory +3. Database URL: `capedb -x db_url=` or `DB_URL` env var or `alembic.ini` + +**Script Entry Point:** Defined in `pyproject.toml` [tool.poetry.scripts] + +## Database URL Configuration + +Three ways to specify DB URL (higher precedence first): +1. CLI argument: `capedb -x db_url=postgresql://...` +2. Environment variable: `DB_URL=postgresql://...` +3. alembic.ini: `sqlalchemy.url = postgresql://...` + +## Migration Workflow + +### Creating New Migrations + +1. **Modify models:** Edit `models.py` with schema changes +2. **Import new models:** Add imports to `migrations/env.py` if new tables +3. **Generate migration:** + ```bash + alembic revision --autogenerate -m "descriptive message" + ``` +4. **Review migration:** Check generated file in `versions/` directory +5. **Test upgrade:** `capedb upgrade head` on test database +6. **Test downgrade:** `capedb downgrade ` to verify rollback +7. **Commit:** Migration file + model changes together + +### Applying Migrations + +```bash +# Upgrade to latest +capedb upgrade head + +# Check current version +capedb current --verbose + +# Downgrade one revision +capedb downgrade -1 +``` + +## Important Notes + +- **Empty DB Required:** Migrations expect empty database to exist before first run +- **Never Bypass:** Always use migrations; never modify schema directly in production +- **Black Excluded:** Migrations directory excluded from Black formatting +- **Migration Files:** Never modify applied migrations; create new ones for changes diff --git a/notes/03-cli-tools.md b/notes/03-cli-tools.md new file mode 100644 index 0000000..4142d53 --- /dev/null +++ b/notes/03-cli-tools.md @@ -0,0 +1,77 @@ +# CLI Tools + +## Overview + +The package provides two command-line scripts, defined in `pyproject.toml` [tool.poetry.scripts]. + +## capedb - Primary Migration Tool + +**Location:** `cape_cod_db/cli/capedb.py` + +**Purpose:** Simplified Alembic wrapper for database migrations. + +**Entry Point:** `main()` function registers custom commands with Alembic's CommandLine + +**Commands:** +- `current(config, verbose=False)` - Display current DB version +- `upgrade(config, revision)` - Upgrade to specified revision +- `downgrade(config, revision)` - Downgrade to specified revision + +**Usage:** +```bash +# Show current version +capedb current +capedb current --verbose + +# Upgrade to latest +capedb upgrade head + +# Downgrade to specific revision +capedb downgrade eecb735a6c3b +``` + +**Configuration:** +- Requires Alembic config (see 02-migrations.md for configuration options) +- Database URL via CLI arg, env var, or config file + +**Implementation:** Uses `alembic.config.CommandLine` and registers custom command functions + +## capedb-app - Direct Table Creation + +**Location:** `cape_cod_db/cli/app.py` + +**Purpose:** Create tables directly from SQLModel metadata WITHOUT migrations. + +**WARNING:** This tool: +- Applies only the current schema (no migration history) +- Does NOT create alembic_version table +- Incompatible with migration-based workflows +- Only useful for fresh databases without migration requirements + +**Usage:** +```bash +# Requires DB_URL environment variable +DB_URL=postgresql://user@localhost/dbname capedb-app +``` + +**Implementation:** Simply calls `database.create_tables()` + +**When to Use:** +- Quick testing/prototyping with throwaway databases +- Projects that don't need migration history + +**When NOT to Use:** +- Production environments +- Any database that needs migration tracking +- Projects using `capedb` for migrations (incompatible) + +## Script Registration + +Both scripts registered in `pyproject.toml`: +```toml +[tool.poetry.scripts] +capedb = "cape_cod_db.cli.capedb:main" +capedb-app = "cape_cod_db.cli.app:main" +``` + +Available after `poetry install` or when package is installed as dependency. diff --git a/notes/04-database-connection.md b/notes/04-database-connection.md new file mode 100644 index 0000000..c5f86ed --- /dev/null +++ b/notes/04-database-connection.md @@ -0,0 +1,99 @@ +# Database Connection + +## Overview + +Database connection setup is handled in `cape_cod_db/database.py`. + +## Connection Configuration + +**Location:** `database.py:5-38` + +### URL Resolution Logic + +Attempts to find database URL in this order: + +1. **Alembic Config (primary):** + - Imports from `migrations.env` config object + - Gets URL from `config.get_main_option("sqlalchemy.url")` + - Used when running migration commands + +2. **Environment Variable (fallback):** + - If Alembic config not available (AttributeError) + - Checks `DB_URL` environment variable + - Used when using ORM directly (e.g., in API lambdas) + +3. **Error on Failure:** + - If no URL found, logs error and exits with code 1 + - Message: "DB_URL is not configured..." + +### Engine Creation + +**Location:** `database.py:38` + +```python +engine = create_engine(db_url) +``` + +- Uses SQLModel's `create_engine()` wrapper around SQLAlchemy +- Engine instance exported for use in other modules +- Logging shows configured database URL (sanitize in production) + +## Usage Patterns + +### In Migration Context + +```python +# database.py automatically gets URL from alembic config +from cape_cod_db import database as db + +# Engine already configured +with Session(db.engine) as session: + # ... ORM operations +``` + +### In Application Context + +```bash +# Set environment variable +export DB_URL="postgresql://user:pass@localhost/cape_env_db" + +# Or use .env file (not in repo) +DB_URL=postgresql://user:pass@localhost/cape_env_db +``` + +```python +# database.py falls back to DB_URL env var +from cape_cod_db import database as db +from sqlmodel import Session + +with Session(db.engine) as session: + # ... ORM operations +``` + +## Direct Table Creation + +**Location:** `database.py:41-46` + +```python +def create_tables(): + """Create tables on the DB pointed to by engine. + + Expects empty database to exist. + """ + SQLModel.metadata.create_all(engine) +``` + +- Used by `capedb-app` CLI tool +- Bypasses migrations (creates current schema only) +- Does NOT create alembic_version table + +## Security Considerations + +- **Never log full DB URLs in production** (contains credentials) +- Current implementation logs URL at INFO level (line 36) - needs sanitization for production +- No credentials stored in code +- Use environment variables or secure config management + +## PostgreSQL Setup + +See README.md for development PostgreSQL setup with proper permissions. diff --git a/notes/05-development.md b/notes/05-development.md new file mode 100644 index 0000000..de6fafd --- /dev/null +++ b/notes/05-development.md @@ -0,0 +1,182 @@ +# Development Workflow + +## Setup + +### Prerequisites + +- Python 3.10+ +- Poetry 2.3+ +- PostgreSQL 18.x (for local development) +- mise en place or asdf (optional, for version management) + +### Installation + +```bash +# Install dependencies +poetry install + +# Activate virtual environment +poetry shell +``` + +### Local PostgreSQL Setup + +See README.md for detailed PostgreSQL 18.x setup including: +- User and group creation +- Database permissions +- Development-only security settings + +## Code Quality Tools + +### Linting: Ruff + +**Check formatting:** +```bash +ruff check cape_cod_db/ +``` + +**Auto-format:** +```bash +ruff format cape_cod_db/ +``` + +**Configuration:** `pyproject.toml` [tool.ruff] +- Line length: 80 + +### Formatting: Black + +**Configuration:** `pyproject.toml` [tool.black] +- Line length: 80 +- **Excludes:** `cape_cod_db/migrations` (generated code) + +### Import Sorting: isort + +**Configuration:** `pyproject.toml` [tool.isort] +- Profile: black +- Line length: 80 +- **Excludes:** `cape_cod_db/migrations` + +### Type Checking: Pyright + +```bash +pyright +``` + +**Configuration:** `pyrightconfig.json` at repo root + +## Testing + +**Status:** TBD - no test framework currently configured + +**Before assuming test commands:** +- Check README.md +- Search codebase for test configuration +- Never assume pytest/unittest without verification + +## Code Conventions + +### Python Style + +1. **Type Hints:** Always use Python 3.10+ syntax + - Use `int | None` (not `Optional[int]`) + - Type all function parameters and returns + +2. **Line Length:** 80 characters (Black/Ruff enforced) + +3. **Imports:** + - Standard library first + - Third-party second + - Local imports last + - Sorted by isort (black profile) + +### Database Patterns + +1. **Model Inheritance:** All tables inherit from `CapeModel` + +2. **Migration Required:** Never bypass Alembic for schema changes + +3. **Model Registration:** Import all models in `migrations/env.py` + +4. **No PII/PHI Logging:** + - Never log sensitive user data + - Sanitize `__repr__` methods (see User model TODO) + +5. **Security:** + - Never commit database URLs, passwords, or credentials + - Use environment variables for sensitive config + - Dev-only PostgreSQL setup documented in README + +## Development Database Operations + +### Using Python REPL + +```python +from sqlmodel import select, Session +from cape_cod_db import database as db +from cape_cod_db import models + +# Create record +with Session(db.engine) as session: + usr = models.User( + first_name="First", + last_name="Last", + email="fl@fakeemail.test" + ) + session.add(usr) + session.commit() + +# Query records +with Session(db.engine) as session: + stmnt = select(models.User) + res = session.exec(stmnt) + for u in res.all(): + print(u) + +# Update record +with Session(db.engine) as session: + usr_stmnt = select(models.User).where( + models.User.first_name == "First" + ) + usr = session.exec(usr_stmnt).first() + usr.first_name = "Updated" + session.add(usr) + session.commit() +``` + +### Using psql + +```bash +psql cape_env_db + +# Show tables +\d+ + +# Describe table +\d user + +# Query +SELECT * FROM public.user; + +# Check migration version +SELECT * FROM alembic_version; +``` + +## Git Workflow + +- Commit migrations with corresponding model changes +- Never modify applied migrations (create new ones) +- Keep AGENTS.md and notes/ synchronized with code changes + +## Pre-commit Hooks + +**Configuration:** `.pre-commit-config.yaml` + +Check configuration file for enabled hooks. + +## Configuration Files + +- `.editorconfig` - Editor settings +- `.prettierrc.yaml` - Prettier formatting (for non-Python files) +- `.typos.toml` - Typo checking configuration +- `.mise.toml` - mise en place tool versions +- `poetry.toml` - Poetry configuration diff --git a/notes/06-authorization-design.md b/notes/06-authorization-design.md new file mode 100644 index 0000000..38d8f63 --- /dev/null +++ b/notes/06-authorization-design.md @@ -0,0 +1,530 @@ +# Authorization Design + +## Overview + +Authorization layer using ABAC (Attribute-Based Access Control) with OPA (Open Policy Agent) for policy decisions. + +## Status + +**Last Updated**: 2026-06-24 + +**Confirmed Decisions**: +- ✅ ABAC with OPA (vs. RBAC/TBAC) +- ✅ Generic Resource model (vs. AWS-specific tables) +- ✅ Resource records only (no S3 path arrays on Tributary) +- ✅ OPA bundle pull model for data sync +- ✅ Pulumi outputs → JSON file → Ansible sync script +- ⏳ UserAttribute table timing (under review: add now vs. post-demo) + +**Implementation Status**: +- 🔄 Database schema design (in progress) +- ⏹️ OPA policies (not started, design in `.design/`) +- ⏹️ Pulumi sync (not started, design in `.design/`) +- ⏹️ Ansible sync script (not started, design in `.design/`) + +## Timeline & Scope + +- **Demo Deadline**: 4 weeks +- **Demo Focus**: Internal users with tributary-based S3 access +- **Future**: External users via SAML federation +- **Resources**: Limited, multiple parallel efforts +- **Infrastructure**: Pulumi (IaC) + Ansible (environment) in separate repos + +## Use Case: Tributary Model + +### Concept +- **Tributaries**: Organizational units within the organization +- **Membership**: Users belong to one or more tributaries +- **Data Ownership**: Tributaries produce data → AWS Data Lake +- **Base Permissions**: Tributary membership grants S3 read/write, EC2 access +- **Flexibility**: Org restructures = attribute updates, not schema changes + +### Demo MVP Use Case +**Primary Flow**: Users select S3 upload location based on tributary membership + +1. User wants to upload data +2. Query API: "What S3 buckets can this user write to?" +3. User selects from allowed locations (based on tributary membership) +4. Upload request gated by OPA policy decision +5. **Stretch Goal**: Download reports with similar flow + +**Current Problem**: Hard-coded single S3 location (not viable) + +## Architecture Decision: ABAC + OPA (CONFIRMED) + +### Why ABAC over RBAC/TBAC + +**Decision**: Use ABAC (Attribute-Based Access Control) with OPA, not RBAC or pure tributary-based access control. + +**Rationale**: +1. Tributary model fits ABAC naturally (membership is a user attribute) +2. External users need attribute-based rules (future requirement) +3. Document-based access grants → Rego policies (flexible policy language) +4. Flexible for complex future scenarios without schema changes +5. Separation of concerns (policies in OPA, not hardcoded in app) +6. Audit-friendly (policy versioning in OPA bundles) +7. Team has skills for both approaches +8. Handles non-tributary users (admins, auditors) via user attributes + +**What This Means**: +- Authorization decisions evaluate user attributes (tributary membership, admin status, clearance level, etc.) +- Policies are NOT fixed role-permission mappings (no "Engineer role has S3 write permission") +- Policies evaluate attributes dynamically: "Allow if user.tributaries contains resource.owner_tributary" + +### Resource Model Decision: Generic (CONFIRMED) + +**Decision**: Single generic `Resource` table with JSONB attributes, not AWS-specific tables (S3Resource, EC2Instance, etc.). + +**Why Generic over AWS-Specific**: +1. **Platform-agnostic**: Works for AWS, Azure, GCP, on-prem resources +2. **Hundreds of resources**: Managing many resource types (S3, EC2, Lambda, Glue, etc.) +3. **Future-proof**: Platform migrations don't require schema changes +4. **Single source of truth**: All resources in one table +5. **Easy to extend**: New resource types = insert rows, not schema migrations +6. **OPA-friendly**: Generic attributes map naturally to ABAC policies + +**Trade-offs Accepted**: +- Lose type safety (no explicit `bucket_name` column, it's in JSONB) +- JSONB queries required for resource-specific filtering +- PostgreSQL JSONB is mature and performant (GIN indexes support fast queries) + +### Tributary Path Storage: Resource Records Only (CONFIRMED) + +**Decision**: Do NOT add `s3_write_paths` or `s3_read_paths` arrays to Tributary table. All resource access defined via Resource records. + +**Why Resource Records Only**: +1. **Single source of truth**: All authorization data in Resource table +2. **Uniform treatment**: Standard buckets and exceptions handled identically +3. **Flexible metadata**: Each resource can have description, expiration, custom attributes +4. **Simpler OPA policies**: Query one place, not Tributary arrays + Resource records +5. **Easy to extend**: Add resource = insert record, no schema change + +**Implementation Note**: +- Every tributary gets 4 standard Resource records (raw_uploads, clean_uploads, raw_results, clean_results) +- Can be auto-created via application code, database trigger, or sync script + +### Components +- **User Attributes**: Stored in PostgreSQL (UserTributary for memberships, potentially UserAttribute for flexible attributes) +- **Policy Decisions**: OPA evaluates Rego policies +- **Data Sync**: Bundle-based pull model (PostgreSQL → JSON → OPA) +- **Infrastructure**: + - OPA on EC2 (currently v1.4.2, needs upgrade to v1.17.1) + - PostgreSQL in AWS RDS (future, currently local dev) + - API Gateway + Cognito identity provider + - Pulumi (IaC) deploys resources, tags them for sync + - Ansible deploys environment (including database) + +## Database Schema + +### Design Decisions Summary + +1. **Generic Resource Model**: Single `Resource` table for all resource types (S3, EC2, Lambda, applications, etc.) +2. **No Tributary Path Arrays**: Tributaries do NOT have `s3_write_paths` or `s3_read_paths` fields +3. **Resource Records Define Access**: All resource access via Resource table records +4. **4 Standard Buckets**: Every tributary gets 4 Resource records for standard data pipeline buckets + +### Core Tables (Demo Focus) + +#### Tributary +Organizational units that own resources and have user members. + +**Fields:** +- `id` - Primary key +- `name` - Display name (unique, indexed) +- `code` - Short code, e.g., "ENG", "DS" (unique, indexed) +- `description` - Purpose/details (optional) +- `parent_id` - Self-reference for hierarchy (optional, FK to tributary.id) +- `attributes` - JSONB for flexible metadata (department, cost_center, etc.) +- `created_at`, `last_edited` - Timestamps (inherited from CapeModel) + +**Note**: Does NOT include S3 path arrays - all resource access defined via Resource table. + +**4 Standard Buckets Pattern**: +Every tributary has 4 buckets allocated to it: +1. Raw user data uploads (triggers ETL when data placed here) +2. Cleaned user upload data (cataloged in datalake) +3. Raw analysis results (triggers ETL) +4. Cleaned analysis result data (cataloged in datalake) + +These are represented as 4 Resource records, not fields on Tributary. + +#### UserTributary (Join Table) +Many-to-many relationship with roles. + +**Fields:** +- `user_id` - FK to user (CASCADE delete) +- `tributary_id` - FK to tributary (CASCADE delete) +- `role` - "member", "admin", "viewer" (default: "member") +- `granted_at` - When membership started +- `granted_by` - User who granted (FK to user) +- `expires_at` - Optional expiration +- **Primary Key**: (user_id, tributary_id) + +**Indexes:** +- `idx_user_tributary_user` on user_id +- `idx_user_tributary_trib` on tributary_id + +#### S3Resource → Resource (Generic Model) +Platform-agnostic resource registry supporting all resource types. + +**Fields:** +- `id` - Primary key +- `resource_type` - Indexed string: "s3", "ec2", "lambda", "glue_table", "application", etc. +- `resource_identifier` - Unique identifier: ARN, URI, or canonical ID (unique, indexed) +- `display_name` - Human-readable name for UI +- `tributary_id` - Owning tributary (FK to tributary.id, nullable for shared resources) +- `access_type` - "read", "write", "both", "execute", "admin", etc. +- `attributes` - JSONB for resource-specific fields (bucket name, instance ID, etc.) +- `created_at`, `last_edited` - Timestamps (inherited from CapeModel) + +**Indexes:** +- `idx_resource_type` on resource_type (B-tree) +- `idx_resource_identifier` on resource_identifier (B-tree, via UNIQUE) +- `idx_resource_tributary` on tributary_id (B-tree) +- `idx_resource_attributes` on attributes (GIN with jsonb_path_ops) + +**Example Data**: +``` +S3 Bucket: + resource_type: "s3" + resource_identifier: "s3://cape-datalake/eng/uploads/" + display_name: "Engineering Raw Uploads" + tributary_id: 1 + access_type: "write" + attributes: {"bucket": "cape-datalake", "path_prefix": "eng/uploads/", "category": "raw_uploads"} + +EC2 Instance: + resource_type: "ec2" + resource_identifier: "arn:aws:ec2:us-east-1:123456:instance/i-abc123" + display_name: "Engineering Web Server" + tributary_id: 1 + access_type: "ssh" + attributes: {"instance_id": "i-abc123", "instance_type": "t3.medium", "vpc_id": "vpc-xyz"} + +Application: + resource_type: "application" + resource_identifier: "cape-web-app-prod" + display_name: "CAPE Web Application (Production)" + tributary_id: NULL (shared) + access_type: "admin" + attributes: {"url": "https://app.cape.example.com", "environment": "prod"} +``` + +**Note**: This replaces the originally planned "S3Resource" table. Generic model chosen for platform-agnosticism and flexibility. + +### Future Tables (Post-Demo or Under Consideration) + +#### UserAttribute (TIMING UNDER REVIEW) +Flexible attribute storage for advanced ABAC. + +**Status**: Originally planned for post-demo, but under consideration to add now to handle non-tributary users (admins, auditors, students with partial permissions). + +**Purpose**: Store user-level attributes like `is_admin`, `clearance_level`, `system_role` for authorization decisions. + +**Fields:** +- `id` - Primary key +- `user_id` - FK to user.id (CASCADE delete) +- `attribute_key` - Attribute name (e.g., "is_admin", "clearance_level") +- `attribute_value` - Attribute value (string representation) +- `source` - "ad", "saml", "manual", "computed" +- `created_at`, `last_edited` - Timestamps +- **UNIQUE**: (user_id, attribute_key) + +**Use Cases**: +- Organizational admins who don't belong to any tributary +- Security auditors who need read access across all tributaries without membership +- Students with limited admin permissions +- Clearance levels for classified data access (future) +- AD/SAML synced attributes (future) + +**Alternative Considered**: Adding `is_admin` boolean and `system_role` string to User table. Rejected to avoid schema churn (add fields → remove fields post-demo). + +**Decision Pending**: Add now (cleaner long-term) or add post-demo (simpler demo)? + +**Options Analyzed**: + +**Option A - Special "Admin" Tributary**: Create system tributary that admins belong to +- Pros: Works with existing schema +- Cons: Semantic mismatch (admins aren't a tributary), doesn't handle partial permissions + +**Option B - Add UserAttribute Table Now**: Store user-level attributes like `is_admin`, `system_role` +- Pros: Proper ABAC, handles all scenarios, no future refactoring +- Cons: Slightly more complex demo, one more table to implement + +**Option C - Direct User-Resource Grants (AccessGrant Table)**: Grant specific users access outside tributary model +- Pros: Handles exceptions cleanly +- Cons: Complex, doesn't solve "admin needs everything" efficiently + +**Option D - Add Role Fields to User Table**: Add `is_admin` boolean and `system_role` string to User +- Pros: Simplest addition (2 columns), handles full admins +- Cons: Schema churn post-demo (add fields → remove when UserAttribute added) + +**Recommendation**: Option B (add UserAttribute table now) to avoid schema churn and demonstrate full ABAC capability. + +#### PartnerOrganization +External organizations with SAML federation. + +**Fields:** +- `id`, `name` (unique) +- `saml_entity_id` (unique) +- `contact_email` +- `attributes` - JSONB +- `created_at`, `last_edited` + +#### ExternalUser +Users from partner organizations. + +**Fields:** +- `id`, `partner_org_id` (FK) +- `external_id` - ID from partner system +- `email` +- `attributes` - JSONB +- `created_at`, `last_edited` +- **Unique**: (partner_org_id, external_id) + +#### Resource +Generic resource tracking (beyond S3). + +**Status**: MERGED INTO CORE SCHEMA - this is now the main Resource table described above, not a future addition. + +~~**Fields:**~~ +~~- `id`, `resource_type`~~ +~~- `resource_arn` (unique)~~ +~~- `tributary_id` (FK, nullable)~~ +~~- `attributes` - JSONB~~ +~~- `created_at`, `last_edited`~~ + +#### AccessGrant +Exception-based access grants (manual overrides). + +**Fields:** +- `id` +- `grantee_type` - "user", "tributary", "partner_org" +- `grantee_id` - Polymorphic reference +- `resource_id` (FK) or `resource_pattern` (flexible matching) +- `permissions` - JSONB array: ["read", "write", "delete"] +- `granted_by` (FK to user), `granted_at` +- `expires_at` - Optional expiration +- `justification` - Text explanation +- `grant_document_url` - Link to agreement document +- `revoked_at`, `revoked_by` - Audit trail + +## OPA Integration + +### Data Sync: Bundle Strategy (Pull Model) - CONFIRMED + +**Decision**: Use bundle-based pull model for demo. + +**Process:** +1. Python script queries PostgreSQL +2. Generates JSON data snapshot +3. Packages as OPA bundle (tar.gz with data.json + manifest) +4. OPA polls bundle location every 30-60 seconds +5. OPA hot-reloads data automatically + +**Why Bundle over Push:** +- Built-in OPA support (no custom endpoints) +- Lowest maintenance overhead +- Good enough for demo (tributary changes infrequent) +- Easy to upgrade to push/hybrid later if needed +- Fastest implementation (1-2 days) + +**Bundle Location**: +- Local file for demo (simplest) +- S3 bucket for production (future) + +**Demo Requirement**: OPA data sync is nice-to-have but NOT required for demo. Can demonstrate with static bundle or manual bundle generation. + +### Authorization Flow + +``` +1. User Request → Application +2. Application enriches context: + - Query PostgreSQL for user attributes + - Build authorization input +3. Application → OPA Decision Request: + POST /v1/data/cape/authorize + { + "input": { + "user": { + "id": 123, + "tributaries": ["trib-1", "trib-2"], + "attributes": {...} + }, + "action": "read" | "write", + "resource": { + "type": "s3", + "path": "s3://bucket/prefix/file.csv", + "owner_tributary": "trib-1" + } + } + } +4. OPA evaluates Rego policies against data bundle +5. OPA returns: {"result": {"allow": true/false}} +6. Application enforces decision +``` + +### Sample Rego Policies + +See `.scratch/session_context.md:248-297` for detailed policy examples. + +**Key Policies:** +- `user_writeable_resources` - Lists S3 paths user can write to +- `allow` - Authorizes specific read/write actions + +**Policy Logic:** +- Default deny +- Write allowed if user in owning tributary + resource has write access +- Read allowed if user in tributary with read permissions +- Future: Access grant overrides + +## Infrastructure → Database Sync (Pulumi → PostgreSQL) + +### Decision: Pulumi Outputs → JSON File → Ansible Sync Script (CONFIRMED) + +**Problem**: Pulumi deploys hundreds of AWS resources. Database needs metadata about these resources for authorization. How does resource data get from Pulumi state into PostgreSQL? + +**Solution Chosen**: Pulumi exports structured JSON file → Ansible ingestion script populates database + +**Flow**: +1. **Pulumi program** (infrastructure repo): + - Creates AWS resources (S3 buckets, EC2 instances, etc.) + - Tags resources consistently (`tributary`, `resource_category`, `access_type`, etc.) + - Exports stack outputs as JSON: `pulumi stack output tributary_resources --json > tributary-resources.json` + +2. **Intermediate JSON file**: + - Format: JSON (native Pulumi support) + - Structure: Tributaries → Resources with metadata + - Stored: Local file or S3 (accessible to Ansible) + +3. **Ansible deployment** (environment repo): + - Reads JSON file (no Pulumi CLI dependency in Ansible environment) + - Python script transforms JSON → Resource records + - Populates PostgreSQL database + +**Why This Approach**: +- **Decoupled**: Ansible doesn't need Pulumi CLI installed (different deployment systems possible) +- **Flexible**: As these are separate public repos, one can be used without the other if valid JSON file provided +- **Structured data**: Pulumi defines clear schema for exports +- **Auditable**: JSON file can be versioned, inspected, validated + +**Tagging Strategy**: Required for resource discovery and billing (in scope for Pulumi repo, recommendations documented in team discussion summary) + +**For Demo**: May manually create JSON file with ~16 resource records (4 tributaries × 4 buckets) rather than full Pulumi export automation. + +## Implementation Plan (4 Weeks) + +**Note**: This plan covers all repos. cape-cod-db repo is only responsible for database schema (Week 1 database tasks). + +### Week 1: Database + OPA Setup +**Database (cape-cod-db repo)**: +- Create Alembic migrations (Tributary, UserTributary, Resource, optionally UserAttribute) +- Seed dev database with test data (fixtures/test/test_data.sql) +- Update documentation (README, notes/) + +**OPA (infrastructure repo)**: +- Upgrade OPA (v1.4.2 → v1.17.1) +- Configure OPA bundle loading + +**Deliverable**: Database schema ready, OPA upgraded + +### Week 2: Core OPA Policies +**OPA policies (infrastructure or dedicated repo)**: +- Write `user_writeable_resources` policy +- Write `allow` policy for S3 read/write +- Create OPA policy tests +- Setup bundle auto-reload (cron/systemd) + +**Deliverable**: Working OPA policies for S3 authorization +**Deliverable**: Working OPA policies for S3 authorization + +### Week 3: API Integration & Sync Scripts +**API (Pulumi repo - Lambda functions)**: +- API endpoint: `GET /user/upload-locations` +- API endpoint: `POST /data/upload` (OPA-gated) +- Optional: Download endpoints +- Add OPA middleware +- Wire Cognito JWT → user_id + +**Sync Scripts**: +- Pulumi: Export resource metadata to JSON +- Ansible: Python script to ingest JSON → PostgreSQL +- Bundle generator: PostgreSQL → OPA bundle (optional for demo) + +**Deliverable**: Working API with S3 permissions, sync infrastructure + +### Week 4: Testing + Demo Polish +- Integration tests +- Load demo data (real tributary/resource data) +- Documentation +- Optional: Svelte admin UI for tributary membership +- Demo script prep + +**Deliverable**: Polished demo + +## Testing Setup (Local Development) + +**Local PostgreSQL**: PostgreSQL 18.4 on development machine + +**Database Connection**: +```bash +# From .env file +DB_URL="postgresql:///cape_env_db" + +# Test connection +source .env && psql "$DB_URL" -c "SELECT version();" +``` + +**Running Migrations**: +```bash +# Current migration state +source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" current --verbose + +# Apply migrations +source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" upgrade head + +# Downgrade +source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" downgrade -1 +``` + +**Loading Test Data**: +```bash +source .env && psql "$DB_URL" < fixtures/test/test_data.sql +``` + +## Demo Data Structure + +See `.scratch/session_context.md:340-364` for complete demo data SQL. + +**Example Setup:** +- **Tributaries**: Engineering (ENG), Data Science (DS), Operations (OPS) +- **S3 Resources**: + - Write: eng/uploads/, ds/uploads/ + - Read: eng/reports/, shared/ +- **Memberships**: + - Alice: ENG admin + DS member + - Bob: DS admin + - Charlie: OPS member + +## Outstanding Questions + +**Updated**: 2026-06-24 + +1. **UserAttribute Table Timing**: Add now (with core tables) or post-demo? Decision pending. +2. **S3 Bucket Structure**: What's the actual naming/path structure for the 4 standard buckets per tributary? + - Same bucket with different prefixes? `s3://cape-datalake/raw-uploads/{tributary-code}/` + - Different buckets per tributary? `s3://cape-raw-uploads-{tributary-code}/` +3. **Access Type Mapping**: How to determine `access_type` for resources? + - From Pulumi tags? (`access_type=write`) + - From naming convention? (raw-uploads → write, clean-uploads → read) + - Hardcoded mapping in sync script? +4. **Cognito Attributes**: What user attributes in JWT? (email, user_id, groups?) +5. **OPA Bundle**: Local file or S3 for demo? +6. **Demo Audience**: Internal stakeholders or end users? + +## Compliance & Audit (Future) + +- Audit trail of policy changes (OPA bundle versioning) +- Track authorization decisions (OPA decision logs) +- Access grant justifications (access_grant table) +- Critical for production, not demo scope diff --git a/notes/README.md b/notes/README.md new file mode 100644 index 0000000..2b4742a --- /dev/null +++ b/notes/README.md @@ -0,0 +1,23 @@ +# cape-cod-db Context Notes + +This directory contains project context documentation for the cape-cod-db project. + +## Purpose + +These notes help agents and developers quickly understand the project structure, architecture, and current state without re-reading all code. + +## Structure + +- **00-overview.md**: High-level project description, purpose, and architecture +- **01-models.md**: Database models and ORM patterns +- **02-migrations.md**: Alembic migration system and workflow +- **03-cli-tools.md**: Command-line scripts and their usage +- **04-database-connection.md**: Database connection setup and configuration +- **05-development.md**: Development workflow, testing, and conventions + +## Maintenance + +When making changes to code: +1. Update relevant context notes in the same session +2. Create new note files if introducing new subsystems +3. Keep notes synchronized with code state From 30eb5ce97f10f79617ad49a6b12786f0e5882c67 Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:23:14 -0400 Subject: [PATCH 03/10] wiring up list/format tools that were missing --- poetry.lock | 271 ++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 9 ++ pyrightconfig.json | 2 +- 3 files changed, 278 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8bd0a9f..9c0df4b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,6 +33,87 @@ files = [ {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] +[[package]] +name = "black" +version = "26.5.1" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "black-26.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9942db8888e06943c5dde66ca0037dcff82a2a4ec1ad0ada9e0d2ee9d9823893"}, + {file = "black-26.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:89c93167a74d3a75dfaa38a5c7cca015537d5820dd7f17d63267d674a61cae90"}, + {file = "black-26.5.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22f2cd76d069cc54c71f10360744ba8983fbb616903b4304a85b734915c8e1b4"}, + {file = "black-26.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:87ed5c6f450580a2f6790bc7cbfb016dfc73bc750249762268a3695361315eef"}, + {file = "black-26.5.1-cp310-cp310-win_arm64.whl", hash = "sha256:58b4bd92cf88aacf83d88479c8f9caee044b1ec55f2451a337354a7ea2590a22"}, + {file = "black-26.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:96ae2c733b2aabdd9986e2c5df628ff3473676cd1c5faded1ff496cf6d74083c"}, + {file = "black-26.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0e48b87e03bf109288e55cfceadcfa15ff5470aca2851a851950ed2926f450d7"}, + {file = "black-26.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5119fa92ae61f786e8c3662fd60aece1d0a2dd5cca5d0c79417a95e7a4272a59"}, + {file = "black-26.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:30d3c14661f2792e9142cce3eeeb1cbc175b3eb5f733be0c8eeb99651e52b0c3"}, + {file = "black-26.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:1ef92b76f7733f282fd096ea406200b5a286c42947412b0eaff3a74e3616cefe"}, + {file = "black-26.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8"}, + {file = "black-26.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217"}, + {file = "black-26.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d"}, + {file = "black-26.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264"}, + {file = "black-26.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418"}, + {file = "black-26.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3"}, + {file = "black-26.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0"}, + {file = "black-26.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294"}, + {file = "black-26.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a"}, + {file = "black-26.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52"}, + {file = "black-26.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168"}, + {file = "black-26.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3"}, + {file = "black-26.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18"}, + {file = "black-26.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50"}, + {file = "black-26.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae"}, + {file = "black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2"}, + {file = "black-26.5.1.tar.gz", hash = "sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=1.0.0" +platformdirs = ">=2" +pytokens = ">=0.4.0,<0.5.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.10)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2) ; sys_platform != \"win32\"", "winloop (>=0.5.0) ; sys_platform == \"win32\""] + +[[package]] +name = "click" +version = "8.4.2" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, + {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +markers = "platform_system == \"Windows\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + [[package]] name = "greenlet" version = "3.3.2" @@ -101,6 +182,21 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil", "setuptools"] +[[package]] +name = "isort" +version = "8.0.1" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.10.0" +groups = ["dev"] +files = [ + {file = "isort-8.0.1-py3-none-any.whl", hash = "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75"}, + {file = "isort-8.0.1.tar.gz", hash = "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d"}, +] + +[package.extras] +colors = ["colorama"] + [[package]] name = "mako" version = "1.3.10" @@ -220,6 +316,71 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, + {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +files = [ + {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, + {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, +] + +[[package]] +name = "packaging" +version = "26.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, + {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189"}, + {file = "pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a"}, +] + +[package.extras] +hyperscan = ["hyperscan (>=0.7)"] +optional = ["typing-extensions (>=4)"] +re2 = ["google-re2 (>=1.1)"] + +[[package]] +name = "platformdirs" +version = "4.10.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, +] + [[package]] name = "psycopg2-binary" version = "2.9.11" @@ -453,6 +614,110 @@ files = [ [package.dependencies] typing-extensions = ">=4.14.1" +[[package]] +name = "pyright" +version = "1.1.411" +description = "Command line wrapper for pyright" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "pyright-1.1.411-py3-none-any.whl", hash = "sha256:dc7c72a8e2700c55baa127554040e067041ea53ccfd50bf96308cc4291c7d5d9"}, + {file = "pyright-1.1.411.tar.gz", hash = "sha256:d885a0551f2e763b089a02702174e7f4ba77548cddabc972ab86d1f7f1b0f998"}, +] + +[package.dependencies] +nodeenv = ">=1.6.0" +typing-extensions = ">=4.1" + +[package.extras] +all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] +nodejs = ["nodejs-wheel-binaries"] + +[[package]] +name = "pytokens" +version = "0.4.1" +description = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5"}, + {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe"}, + {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c"}, + {file = "pytokens-0.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7"}, + {file = "pytokens-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2"}, + {file = "pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440"}, + {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc"}, + {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d"}, + {file = "pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16"}, + {file = "pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6"}, + {file = "pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083"}, + {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1"}, + {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1"}, + {file = "pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9"}, + {file = "pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68"}, + {file = "pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b"}, + {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f"}, + {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1"}, + {file = "pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4"}, + {file = "pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78"}, + {file = "pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321"}, + {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa"}, + {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d"}, + {file = "pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324"}, + {file = "pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9"}, + {file = "pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb"}, + {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3"}, + {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975"}, + {file = "pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a"}, + {file = "pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918"}, + {file = "pytokens-0.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc"}, + {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009"}, + {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1"}, + {file = "pytokens-0.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6"}, + {file = "pytokens-0.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037"}, + {file = "pytokens-0.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3"}, + {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1"}, + {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db"}, + {file = "pytokens-0.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1"}, + {file = "pytokens-0.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a"}, + {file = "pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de"}, + {file = "pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a"}, +] + +[package.extras] +dev = ["black", "build", "mypy", "pytest", "pytest-cov", "setuptools", "tox", "twine", "wheel"] + +[[package]] +name = "ruff" +version = "0.15.19" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.15.19-py3-none-linux_armv6l.whl", hash = "sha256:922d1eb283161564759bd49f507e91dc6112c15da8bd5b84ed714e086243cf86"}, + {file = "ruff-0.15.19-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4d190d8f62a0b94aba8f721116538a9ee29b1e74d26650846ba9b99f0ae21c40"}, + {file = "ruff-0.15.19-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5a2c86ba6870dd415a9d9eb8be94d7924ebec6a26ffc7958ec7ca29d4bff967d"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82b432bc087264aea70fd25ac198918b70bd9e2aa0db4297b0bb91bbfbbc63ce"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8530a09d03b3a8c994f8b559a7dcdabc690bcd3f78ef276c38c83166798ebf56"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87bf21fb3875fe69f0eacc825411657e2e85589cce633c35c0adf1113649c62b"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f9b229cb3ef56ecc2c1c8ebeca64b7a7740ccaef40a9eb097e78dde5a8560b83"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6c754515be7b76afe6e7e62df7776709571bcfc1631183828afcf3bafa869e3"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a498f82e0f4d8904c4e0aea5139cdfac1f39d19a3c51d491292f63a36e83b2e"}, + {file = "ruff-0.15.19-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:d48caa34488fb521fd0ef4aea2b0e8fe758298df044138f0d67b687a6a0d07ed"}, + {file = "ruff-0.15.19-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4171b6613effa9363cd46dd4f75bd1827b6d1b946b5e278ed0c600d305379445"}, + {file = "ruff-0.15.19-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:27c15b2a241dd4d995557949a094fe78b8ad99122a38ccae1595849bcc947b3f"}, + {file = "ruff-0.15.19-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ed03b7862d68f0a8771d50ee129980cbf1b113f96e250b73954bc292f689e0bb"}, + {file = "ruff-0.15.19-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:08143f0685ae278b30727ea72e90c61e5bd9c31b91aac4f5bb989538f73d24b8"}, + {file = "ruff-0.15.19-py3-none-win32.whl", hash = "sha256:8f47f0f92952af2557212bb10cf3e695cd4cf28b2c6e42cdb18ec6c9ebfa19da"}, + {file = "ruff-0.15.19-py3-none-win_amd64.whl", hash = "sha256:efeca47ee3f9d4a7162655a3b8e6ee4a878646044233978d4d2c1ff8cdd914f0"}, + {file = "ruff-0.15.19-py3-none-win_arm64.whl", hash = "sha256:6c6b607466e47349332eb1d9be52fb1467423fc07c217341af41cd0f3f0573be"}, + {file = "ruff-0.15.19.tar.gz", hash = "sha256:edc27f7172a93b32b102687009d6a588508815072141543ae603a8b9b0823063"}, +] + [[package]] name = "sqlalchemy" version = "2.0.48" @@ -577,7 +842,7 @@ version = "2.4.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "dev"] markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867"}, @@ -635,7 +900,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -659,4 +924,4 @@ typing-extensions = ">=4.12.0" [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "88c72aa5a8140b3d2a40f0fb6307920af40be74748d9aba4183e5027dcbd725b" +content-hash = "e5385549bcebadbd7ce6f2ec527e3f21329cfaa40afaeb7f0aa22d7b33402f7a" diff --git a/pyproject.toml b/pyproject.toml index 4e6c9aa..ab4cf1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,3 +33,12 @@ line_length = 80 [tool.ruff] line-length = 80 +extend-exclude = ["cape_cod_db/migrations"] + +[dependency-groups] +dev = [ + "ruff (>=0.15.19,<0.16.0)", + "pyright (>=1.1.411,<2.0.0)", + "black (>=26.5.1,<27.0.0)", + "isort (>=8.0.1,<9.0.0)" +] diff --git a/pyrightconfig.json b/pyrightconfig.json index 3505172..db8c463 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,5 +1,5 @@ { "autoImportCompletions": true, "typeCheckingMode": "basic", - "ignore": ["cape_cod_db/migrations/**"] + "ignore": ["cape_cod_db/migrations/**", "dist/**"] } From 6b6a4d437840e32f10a885d4a5eb98a9d0e92179 Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:24:48 -0400 Subject: [PATCH 04/10] new models for start of environment authz with ABAC --- cape_cod_db/models.py | 114 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/cape_cod_db/models.py b/cape_cod_db/models.py index 47a0f1b..2556c04 100644 --- a/cape_cod_db/models.py +++ b/cape_cod_db/models.py @@ -1,6 +1,8 @@ from datetime import datetime, timezone -from sqlmodel import Field, SQLModel +from sqlmodel import Field, SQLModel, Column +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy import Text # NOTE: For now we're going to keep all models in one module. Should this get # painful we will look at splitting this out. Both ways of doing models @@ -45,3 +47,113 @@ def __repr__(self): f"first_name='{self.first_name}', " f"last_name='{self.last_name}'>" ) + + +class Tributary(CapeModel, table=True): + """Organizational unit that owns resources and has user members. + + Tributaries represent organizational units (teams, departments, projects) + that produce data and own AWS/infrastructure resources. Users belong to + one or more tributaries, and this membership grants base permissions. + + Every tributary gets 4 standard S3 buckets for the data pipeline: + 1. Raw user data uploads (triggers ETL) + 2. Cleaned user upload data (cataloged in datalake) + 3. Raw analysis results (triggers ETL) + 4. Cleaned analysis result data (cataloged) + + These buckets are represented as Resource records, not fields on this model. + """ + + id: int | None = Field(default=None, primary_key=True) + name: str = Field(unique=True, index=True) + code: str = Field(unique=True, index=True) + description: str | None = Field(default=None, sa_column=Column(Text)) + parent_id: int | None = Field(default=None, foreign_key="tributary.id") + attributes: dict = Field( + default_factory=dict, + sa_column=Column(JSONB, nullable=False, server_default='{}') + ) + + +class UserTributary(SQLModel, table=True): + """Many-to-many relationship between users and tributaries with role. + + Represents user membership in a tributary. A user can belong to multiple + tributaries with different roles. This is the core attribute for ABAC + authorization decisions. + + Roles: + - "member": Standard access to tributary resources + - "admin": Elevated access, can manage tributary membership + - "viewer": Read-only access + """ + + user_id: int = Field( + foreign_key="user.id", + primary_key=True + ) + tributary_id: int = Field( + foreign_key="tributary.id", + primary_key=True + ) + role: str = Field(default="member") + granted_at: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc), + nullable=False + ) + granted_by: int | None = Field(default=None, foreign_key="user.id") + expires_at: datetime | None = None + + +class Resource(CapeModel, table=True): + """Generic resource that can be authorized via ABAC. + + Platform-agnostic resource registry supporting AWS resources (S3, EC2, + Lambda, Glue, etc.) and non-AWS resources (applications, databases). + Uses JSONB attributes for resource-specific fields. + + Examples: + - S3: resource_identifier="s3://bucket/path", + attributes={"bucket": "cape-datalake", "path_prefix": "eng/uploads/", + "category": "raw_uploads"} + - EC2: resource_identifier="arn:aws:ec2:...:instance/i-abc", + attributes={"instance_id": "i-abc", "instance_type": "t3.medium"} + - App: resource_identifier="cape-web-app-prod", + attributes={"url": "https://app.example.com", "environment": "prod"} + """ + + id: int | None = Field(default=None, primary_key=True) + resource_type: str = Field(index=True) + resource_identifier: str = Field(unique=True, index=True) + display_name: str + tributary_id: int | None = Field(default=None, foreign_key="tributary.id") + access_type: str + attributes: dict = Field( + default_factory=dict, + sa_column=Column(JSONB, nullable=False, server_default='{}') + ) + + +class UserAttribute(CapeModel, table=True): + """Flexible attribute storage for advanced ABAC. + + Stores user-level attributes for authorization decisions. Common use cases: + - User lifecycle: user_status="quarantine"|"active"|"suspended"|"deactivated" + - Admin privileges: is_admin="true" + - Clearance levels: clearance_level="secret" + - System roles: system_role="security_auditor" + - AD/SAML synced attributes: source="ad" or "saml" + + Attributes are deployment-specific. The system provides the capability; + specific workflows determine which attributes to use. + """ + + id: int | None = Field(default=None, primary_key=True) + user_id: int = Field( + foreign_key="user.id", + index=True + ) + attribute_key: str + attribute_value: str + source: str | None = None From 926c0dc1d645e3e18e5ce41b36e459a0027e297c Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:25:35 -0400 Subject: [PATCH 05/10] migration for new abac authz tables --- cape_cod_db/migrations/env.py | 2 +- ...dd_authorization_tables_tributary_user_.py | 128 ++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py diff --git a/cape_cod_db/migrations/env.py b/cape_cod_db/migrations/env.py index fbab9c9..f748635 100644 --- a/cape_cod_db/migrations/env.py +++ b/cape_cod_db/migrations/env.py @@ -7,7 +7,7 @@ from sqlmodel import SQLModel # NOTE: as new table models are added, they need to be imported here. -from cape_cod_db.models import User +from cape_cod_db.models import User, Tributary, UserTributary, Resource, UserAttribute # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py b/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py new file mode 100644 index 0000000..6cb703d --- /dev/null +++ b/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py @@ -0,0 +1,128 @@ +"""add authorization tables (tributary, user_tributary, resource, user_attribute) + +Revision ID: 6919c61ea401 +Revises: 6001985fea71 +Create Date: 2026-06-25 11:10:27.936819 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +import sqlmodel + +# revision identifiers, used by Alembic. +revision: str = '6919c61ea401' +down_revision: Union[str, Sequence[str], None] = '6001985fea71' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tributary', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('last_edited', sa.DateTime(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('code', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('parent_id', sa.Integer(), nullable=True), + sa.Column('attributes', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False), + sa.ForeignKeyConstraint(['parent_id'], ['tributary.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('tributary', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_tributary_code'), ['code'], unique=True) + batch_op.create_index(batch_op.f('ix_tributary_name'), ['name'], unique=True) + + op.create_table('resource', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('last_edited', sa.DateTime(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('resource_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('resource_identifier', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('display_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('tributary_id', sa.Integer(), nullable=True), + sa.Column('access_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attributes', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False), + sa.ForeignKeyConstraint(['tributary_id'], ['tributary.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('resource', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_resource_resource_identifier'), ['resource_identifier'], unique=True) + batch_op.create_index(batch_op.f('ix_resource_resource_type'), ['resource_type'], unique=False) + + # Add GIN index for JSONB attributes column on resource table + op.create_index( + 'ix_resource_attributes', + 'resource', + ['attributes'], + unique=False, + postgresql_using='gin', + postgresql_ops={'attributes': 'jsonb_path_ops'} + ) + + op.create_table('userattribute', + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('last_edited', sa.DateTime(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('attribute_key', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attribute_value', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('source', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user_id', 'attribute_key', name='uq_userattribute_user_id_attribute_key') + ) + with op.batch_alter_table('userattribute', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_userattribute_user_id'), ['user_id'], unique=False) + + op.create_table('usertributary', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('tributary_id', sa.Integer(), nullable=False), + sa.Column('role', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('granted_at', sa.DateTime(), nullable=False), + sa.Column('granted_by', sa.Integer(), nullable=True), + sa.Column('expires_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['granted_by'], ['user.id'], ), + sa.ForeignKeyConstraint(['tributary_id'], ['tributary.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('user_id', 'tributary_id') + ) + with op.batch_alter_table('usertributary', schema=None) as batch_op: + batch_op.create_index('ix_usertributary_user_id', ['user_id'], unique=False) + batch_op.create_index('ix_usertributary_tributary_id', ['tributary_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('usertributary', schema=None) as batch_op: + batch_op.drop_index('ix_usertributary_tributary_id') + batch_op.drop_index('ix_usertributary_user_id') + + op.drop_table('usertributary') + with op.batch_alter_table('userattribute', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_userattribute_user_id')) + + op.drop_table('userattribute') + + # Drop GIN index on resource attributes + op.drop_index('ix_resource_attributes', table_name='resource') + + with op.batch_alter_table('resource', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_resource_resource_type')) + batch_op.drop_index(batch_op.f('ix_resource_resource_identifier')) + + op.drop_table('resource') + with op.batch_alter_table('tributary', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_tributary_name')) + batch_op.drop_index(batch_op.f('ix_tributary_code')) + + op.drop_table('tributary') + # ### end Alembic commands ### From 0818b06b88c85ed00bb9cabfc5cc6c5321ec0168 Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:26:09 -0400 Subject: [PATCH 06/10] linter/formatter fixen --- cape_cod_db/cli/app.py | 1 - cape_cod_db/database.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cape_cod_db/cli/app.py b/cape_cod_db/cli/app.py index 02ad76e..ed69436 100644 --- a/cape_cod_db/cli/app.py +++ b/cape_cod_db/cli/app.py @@ -1,5 +1,4 @@ from cape_cod_db.database import create_tables -from cape_cod_db.models import User # WARNING: Usage of this script only applies the most recent schema, with 0 # respect to migrations and without including any migration tables or diff --git a/cape_cod_db/database.py b/cape_cod_db/database.py index a2c4b50..35e297b 100644 --- a/cape_cod_db/database.py +++ b/cape_cod_db/database.py @@ -14,7 +14,7 @@ # some other source for the DB URL as the config object doesn't exit in # env.py. alembic doesn't play when we're just doing orm things. db_url = config.get_main_option("sqlalchemy.url") -except AttributeError as ae: +except AttributeError: logging.basicConfig() logger = logging.getLogger(__file__) logger.warning( From 3b1c338d620b25be9fcad5d17b6ac34b079b449d Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:26:41 -0400 Subject: [PATCH 07/10] added sinple test data and cleanup sql scripts for testing model changes --- fixtures/test/cleanup_test_data.sql | 82 ++++++++++ fixtures/test/test_data.sql | 245 ++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 fixtures/test/cleanup_test_data.sql create mode 100644 fixtures/test/test_data.sql diff --git a/fixtures/test/cleanup_test_data.sql b/fixtures/test/cleanup_test_data.sql new file mode 100644 index 0000000..7f81b6f --- /dev/null +++ b/fixtures/test/cleanup_test_data.sql @@ -0,0 +1,82 @@ +-- Cleanup Test Data for CAPE Authorization System +-- +-- PURPOSE: Removes test data loaded by fixtures/test/test_data.sql +-- +-- USAGE: +-- psql cape_env_db < fixtures/test/cleanup_test_data.sql +-- +-- WHAT THIS DOES: +-- - Deletes all users with @example.com emails +-- - Deletes tributaries by known test codes +-- - CASCADE deletes automatically remove: +-- - user_tributary memberships (FK CASCADE on user_id, tributary_id) +-- - user_attribute records (FK CASCADE on user_id) +-- - resources owned by deleted tributaries (FK to tributary_id) +-- +-- SAFETY: +-- - Only deletes data matching test patterns (@example.com, known codes) +-- - Uses transaction (can ROLLBACK if something looks wrong) +-- - Verification queries at end show what was removed + +BEGIN; + +-- Count before deletion (for verification) +\echo '' +\echo '=== Before Cleanup ===' +SELECT + (SELECT COUNT(*) FROM "user") as users, + (SELECT COUNT(*) FROM tributary) as tributaries, + (SELECT COUNT(*) FROM usertributary) as memberships, + (SELECT COUNT(*) FROM resource) as resources, + (SELECT COUNT(*) FROM userattribute) as user_attributes; + +-- Delete in proper order to avoid FK constraint violations + +-- First: Set granted_by to NULL for memberships granted by test users +-- (avoids FK violation when deleting users) +UPDATE usertributary SET granted_by = NULL +WHERE granted_by IN (SELECT id FROM "user" WHERE email LIKE '%@example.com'); + +-- Second: Delete resources owned by test tributaries +-- (must happen before deleting tributaries due to FK constraint) +DELETE FROM resource WHERE tributary_id IN ( + SELECT id FROM tributary WHERE code IN ('ENG', 'PLAT-ENG', 'DS', 'OPS') +); + +-- Also delete test resources with no tributary owner +DELETE FROM resource WHERE resource_identifier IN ( + 'arn:aws:ec2:us-east-1:123456789012:instance/i-eng001', + 'cape-web-app-prod', + 's3://cape-datalake/shared/' +); + +-- Third: Delete test tributaries (by known test codes) +-- This will CASCADE delete: +-- - usertributary records (tributary_id FK CASCADE) +DELETE FROM tributary WHERE code IN ('ENG', 'PLAT-ENG', 'DS', 'OPS'); + +-- Fourth: Delete test users (by email pattern) +-- This will CASCADE delete: +-- - usertributary records (user_id FK CASCADE) - any remaining +-- - userattribute records (user_id FK CASCADE) +DELETE FROM "user" WHERE email LIKE '%@example.com'; + +COMMIT; + +-- Verification after deletion +\echo '' +\echo '=== After Cleanup ===' +SELECT + (SELECT COUNT(*) FROM "user") as users, + (SELECT COUNT(*) FROM tributary) as tributaries, + (SELECT COUNT(*) FROM usertributary) as memberships, + (SELECT COUNT(*) FROM resource) as resources, + (SELECT COUNT(*) FROM userattribute) as user_attributes; + +\echo '' +\echo '=== Test Data Cleanup Complete ===' +\echo '' +\echo 'If counts above are not what you expected, you can:' +\echo ' 1. Check what remains: SELECT * FROM "user"; SELECT * FROM tributary;' +\echo ' 2. Re-run cleanup if needed (script is idempotent)' +\echo '' diff --git a/fixtures/test/test_data.sql b/fixtures/test/test_data.sql new file mode 100644 index 0000000..8a125a6 --- /dev/null +++ b/fixtures/test/test_data.sql @@ -0,0 +1,245 @@ +-- Test Data for CAPE Authorization System +-- +-- PURPOSE: Provides realistic test data for development and testing. +-- Includes users, tributaries, memberships, resources, and attributes. +-- +-- LOADING TEST DATA: +-- Method 1 - Permanent (keeps data until manually cleaned): +-- psql cape_env_db < fixtures/test/test_data.sql +-- +-- Method 2 - Temporary (auto-rollback, no cleanup needed): +-- psql cape_env_db < Date: Thu, 25 Jun 2026 12:34:24 -0400 Subject: [PATCH 08/10] updated to cover new tables and some general info about the (early) ABAC implementation --- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2fc2cd8..a01e211 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,40 @@ # CAPE Environment Database Repo -**_TBD_** blah blah repo for the main cape environment database +Repo for the CAPE environment database. This is the main database used by +various resources in CAPE for tracking users, user attributes, and AWS resource +data. + +## Schema Overview + +### Authorization Model + +The database implements an Attribute-Based Access Control (ABAC) system with the +following components: + +- **User**: Core user identity (from Cognito) +- **Tributary**: Logical data compartments with associated AWS resources +- **UserTributary**: Many-to-many junction between users and tributaries +- **Resource**: Platform-agnostic resource definitions (S3 paths, EC2 instances, + etc.) linked to tributaries +- **UserAttribute**: Key-value attributes for users (e.g., `user_status`, + `role`, `department`) + +#### Key Relationships + +- Users belong to multiple tributaries (via `UserTributary`) +- Each tributary has multiple resources (via `Resource.tributary_id`) +- Resources define access patterns: `s3_read`, `s3_write`, `ec2_connect`, etc. +- User attributes handle non-tributary authorization (quarantine, admin roles, + suspension) + +#### Standard Resource Pattern + +Each tributary typically has 4 S3 bucket resources: + +- `raw_uploads` (write access) +- `clean_uploads` (read access) +- `raw_results` (write access) +- `clean_results` (read access) ## Setup @@ -135,6 +169,24 @@ ever. This script requires the `DB_URL` environment variable be set and does not work with alembic at all, so the alembic config is not needed. +### Test Data + +Test data fixtures are provided in `fixtures/test/`: + +- `test_data.sql`: Loads 6 users, 4 tributaries, 12 resources, 9 user attributes +- `cleanup_test_data.sql`: Removes test data in proper dependency order + +```bash +# Load test data (use transaction to allow rollback) +source .env && psql "$DB_URL" -f fixtures/test/test_data.sql + +# Clean up test data +source .env && psql "$DB_URL" -f fixtures/test/cleanup_test_data.sql +``` + +**Important**: When modifying schema or test data, keep both SQL files +synchronized with changes. + ### Play With The DB From repo root in the python repl of your fancy (that has all the dependencies @@ -179,6 +231,26 @@ with Session(db.engine) as session: for u in res.all(): print(u) + +# query authorization data +with Session(db.engine) as session: + # get user's tributaries + user = session.exec(select(models.User).where(models.User.email == "alice@example.com")).first() + tributaries = session.exec( + select(models.Tributary) + .join(models.UserTributary) + .where(models.UserTributary.user_id == user.id) + ).all() + + # get resources for a tributary + resources = session.exec( + select(models.Resource).where(models.Resource.tributary_id == tributaries[0].id) + ).all() + + # get user attributes + attrs = session.exec( + select(models.UserAttribute).where(models.UserAttribute.user_id == user.id) + ).all() ``` If you prefer `psql` (**_NOTE:_** this assumes you have all the perms on the From bc3659028027ff021c320207b8d93bf91b7da214 Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 12:35:13 -0400 Subject: [PATCH 09/10] added ignore for .design directory, where i'm tracking design artifacts for other repos --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b33f8ae..b8388bb 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,6 @@ cython_debug/ # sqlite db *.db *.sqlite + +# Cross-repo implementation plans (not committed) +.design/ From beead6a482cf6dfa6f9cfa8b3772ee129834bf1f Mon Sep 17 00:00:00 2001 From: Drew Pihera Date: Thu, 25 Jun 2026 13:38:34 -0400 Subject: [PATCH 10/10] Fix pre-commit issues and exclude notes/ from linting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix typo in workflow: Publsih → Publish - Add notes/ to pre-commit global exclude (LLM context only) - Add notes/ to typos exclude list - Accept auto-fixes: trailing whitespace, black formatting, isort --- .github/workflows/cape.yml | 2 +- .pre-commit-config.yaml | 2 ++ .typos.toml | 2 +- AGENTS.md | 2 +- cape_cod_db/migrations/env.py | 8 +++++- .../6001985fea71_add_email_to_user_table.py | 4 +-- ...dd_authorization_tables_tributary_user_.py | 13 +++++----- .../eecb735a6c3b_create_user_table.py | 1 - cape_cod_db/models.py | 26 ++++++------------- fixtures/test/cleanup_test_data.sql | 4 +-- fixtures/test/test_data.sql | 20 +++++++------- 11 files changed, 39 insertions(+), 45 deletions(-) diff --git a/.github/workflows/cape.yml b/.github/workflows/cape.yml index 45bf31e..5de0291 100644 --- a/.github/workflows/cape.yml +++ b/.github/workflows/cape.yml @@ -24,7 +24,7 @@ jobs: name: General uses: cape-ph/.github/.github/workflows/general_checks.yml@v1 publish: - name: Publsih to PyPI + name: Publish to PyPI needs: python runs-on: ubuntu-latest if: ${{ github.event_name == 'release' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7130391..8463128 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +exclude: ^notes/ + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 diff --git a/.typos.toml b/.typos.toml index cfc65ab..7e4280d 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,6 +1,6 @@ # See https://github.com/crate-ci/typos/blob/master/docs/reference.md to configure typos [files] -extend-exclude = ["roles/*/files"] +extend-exclude = ["roles/*/files", "notes/"] [default.extend-words] Furst = "Furst" diff --git a/AGENTS.md b/AGENTS.md index 6aa14e2..7c530a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,7 +50,7 @@ Before marking work as complete, verify: source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" current --verbose source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" upgrade head source .env && capedb -c cape_cod_db/alembic.ini -x db_url="$DB_URL" downgrade -1 - + # Or use alembic directly alembic -c cape_cod_db/alembic.ini revision --autogenerate -m "description" ``` diff --git a/cape_cod_db/migrations/env.py b/cape_cod_db/migrations/env.py index f748635..2f1dc68 100644 --- a/cape_cod_db/migrations/env.py +++ b/cape_cod_db/migrations/env.py @@ -7,7 +7,13 @@ from sqlmodel import SQLModel # NOTE: as new table models are added, they need to be imported here. -from cape_cod_db.models import User, Tributary, UserTributary, Resource, UserAttribute +from cape_cod_db.models import ( + Resource, + Tributary, + User, + UserAttribute, + UserTributary, +) # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/cape_cod_db/migrations/versions/6001985fea71_add_email_to_user_table.py b/cape_cod_db/migrations/versions/6001985fea71_add_email_to_user_table.py index aeb6813..0dd9076 100644 --- a/cape_cod_db/migrations/versions/6001985fea71_add_email_to_user_table.py +++ b/cape_cod_db/migrations/versions/6001985fea71_add_email_to_user_table.py @@ -7,11 +7,9 @@ """ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa - - import sqlmodel +from alembic import op # revision identifiers, used by Alembic. revision: str = '6001985fea71' diff --git a/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py b/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py index 6cb703d..438fe66 100644 --- a/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py +++ b/cape_cod_db/migrations/versions/6919c61ea401_add_authorization_tables_tributary_user_.py @@ -7,11 +7,10 @@ """ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - import sqlmodel +from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision: str = '6919c61ea401' @@ -55,7 +54,7 @@ def upgrade() -> None: with op.batch_alter_table('resource', schema=None) as batch_op: batch_op.create_index(batch_op.f('ix_resource_resource_identifier'), ['resource_identifier'], unique=True) batch_op.create_index(batch_op.f('ix_resource_resource_type'), ['resource_type'], unique=False) - + # Add GIN index for JSONB attributes column on resource table op.create_index( 'ix_resource_attributes', @@ -105,16 +104,16 @@ def downgrade() -> None: with op.batch_alter_table('usertributary', schema=None) as batch_op: batch_op.drop_index('ix_usertributary_tributary_id') batch_op.drop_index('ix_usertributary_user_id') - + op.drop_table('usertributary') with op.batch_alter_table('userattribute', schema=None) as batch_op: batch_op.drop_index(batch_op.f('ix_userattribute_user_id')) op.drop_table('userattribute') - + # Drop GIN index on resource attributes op.drop_index('ix_resource_attributes', table_name='resource') - + with op.batch_alter_table('resource', schema=None) as batch_op: batch_op.drop_index(batch_op.f('ix_resource_resource_type')) batch_op.drop_index(batch_op.f('ix_resource_resource_identifier')) diff --git a/cape_cod_db/migrations/versions/eecb735a6c3b_create_user_table.py b/cape_cod_db/migrations/versions/eecb735a6c3b_create_user_table.py index 221434d..adf18c9 100644 --- a/cape_cod_db/migrations/versions/eecb735a6c3b_create_user_table.py +++ b/cape_cod_db/migrations/versions/eecb735a6c3b_create_user_table.py @@ -10,7 +10,6 @@ import sqlalchemy as sa import sqlmodel - from alembic import op # revision identifiers, used by Alembic. diff --git a/cape_cod_db/models.py b/cape_cod_db/models.py index 2556c04..c14fa1e 100644 --- a/cape_cod_db/models.py +++ b/cape_cod_db/models.py @@ -1,8 +1,8 @@ from datetime import datetime, timezone -from sqlmodel import Field, SQLModel, Column -from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy import Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import Column, Field, SQLModel # NOTE: For now we're going to keep all models in one module. Should this get # painful we will look at splitting this out. Both ways of doing models @@ -72,7 +72,7 @@ class Tributary(CapeModel, table=True): parent_id: int | None = Field(default=None, foreign_key="tributary.id") attributes: dict = Field( default_factory=dict, - sa_column=Column(JSONB, nullable=False, server_default='{}') + sa_column=Column(JSONB, nullable=False, server_default="{}"), ) @@ -89,18 +89,11 @@ class UserTributary(SQLModel, table=True): - "viewer": Read-only access """ - user_id: int = Field( - foreign_key="user.id", - primary_key=True - ) - tributary_id: int = Field( - foreign_key="tributary.id", - primary_key=True - ) + user_id: int = Field(foreign_key="user.id", primary_key=True) + tributary_id: int = Field(foreign_key="tributary.id", primary_key=True) role: str = Field(default="member") granted_at: datetime = Field( - default_factory=lambda: datetime.now(timezone.utc), - nullable=False + default_factory=lambda: datetime.now(timezone.utc), nullable=False ) granted_by: int | None = Field(default=None, foreign_key="user.id") expires_at: datetime | None = None @@ -131,7 +124,7 @@ class Resource(CapeModel, table=True): access_type: str attributes: dict = Field( default_factory=dict, - sa_column=Column(JSONB, nullable=False, server_default='{}') + sa_column=Column(JSONB, nullable=False, server_default="{}"), ) @@ -150,10 +143,7 @@ class UserAttribute(CapeModel, table=True): """ id: int | None = Field(default=None, primary_key=True) - user_id: int = Field( - foreign_key="user.id", - index=True - ) + user_id: int = Field(foreign_key="user.id", index=True) attribute_key: str attribute_value: str source: str | None = None diff --git a/fixtures/test/cleanup_test_data.sql b/fixtures/test/cleanup_test_data.sql index 7f81b6f..fc0608a 100644 --- a/fixtures/test/cleanup_test_data.sql +++ b/fixtures/test/cleanup_test_data.sql @@ -23,7 +23,7 @@ BEGIN; -- Count before deletion (for verification) \echo '' \echo '=== Before Cleanup ===' -SELECT +SELECT (SELECT COUNT(*) FROM "user") as users, (SELECT COUNT(*) FROM tributary) as tributaries, (SELECT COUNT(*) FROM usertributary) as memberships, @@ -66,7 +66,7 @@ COMMIT; -- Verification after deletion \echo '' \echo '=== After Cleanup ===' -SELECT +SELECT (SELECT COUNT(*) FROM "user") as users, (SELECT COUNT(*) FROM tributary) as tributaries, (SELECT COUNT(*) FROM usertributary) as memberships, diff --git a/fixtures/test/test_data.sql b/fixtures/test/test_data.sql index 8a125a6..bdfc8fb 100644 --- a/fixtures/test/test_data.sql +++ b/fixtures/test/test_data.sql @@ -1,5 +1,5 @@ -- Test Data for CAPE Authorization System --- +-- -- PURPOSE: Provides realistic test data for development and testing. -- Includes users, tributaries, memberships, resources, and attributes. -- @@ -48,7 +48,7 @@ INSERT INTO tributary (created_at, last_edited, name, code, description, parent_ -- Insert Platform Engineering as child of Engineering (requires Engineering's ID) INSERT INTO tributary (created_at, last_edited, name, code, description, parent_id, attributes) -SELECT NOW(), NOW(), 'Platform Engineering', 'PLAT-ENG', 'Platform and DevOps team', +SELECT NOW(), NOW(), 'Platform Engineering', 'PLAT-ENG', 'Platform and DevOps team', t.id, '{"department": "technology", "cost_center": "plat-001"}'::jsonb FROM tributary t WHERE t.code = 'ENG'; @@ -120,7 +120,7 @@ WHERE u.email = 'charlie.operator@example.com' AND t.code = 'OPS'; INSERT INTO usertributary (user_id, tributary_id, role, granted_at, granted_by, expires_at) SELECT diana.id, eng.id, 'member', NOW(), alice.id, NULL::timestamp FROM "user" diana, "user" alice, tributary eng -WHERE diana.email = 'diana.multirole@example.com' +WHERE diana.email = 'diana.multirole@example.com' AND alice.email = 'alice.engineer@example.com' AND eng.code = 'ENG' UNION ALL @@ -192,7 +192,7 @@ SELECT id, code, name, parent_id, description FROM tributary ORDER BY id; \echo '' \echo 'User Memberships:' -SELECT +SELECT u.id as user_id, u.first_name || ' ' || u.last_name as user_name, t.code as tributary, @@ -205,7 +205,7 @@ ORDER BY u.id, t.id; \echo '' \echo 'User Attributes:' -SELECT +SELECT u.id as user_id, u.first_name || ' ' || u.last_name as user_name, ua.attribute_key, @@ -217,7 +217,7 @@ ORDER BY u.id, ua.attribute_key; \echo '' \echo 'Resources (first 10):' -SELECT +SELECT r.id, r.resource_type, r.resource_identifier, @@ -230,14 +230,14 @@ LIMIT 10; \echo '' \echo 'Resource count by type:' -SELECT resource_type, COUNT(*) as count -FROM resource -GROUP BY resource_type +SELECT resource_type, COUNT(*) as count +FROM resource +GROUP BY resource_type ORDER BY resource_type; \echo '' \echo '=== Test Data Summary ===' -SELECT +SELECT (SELECT COUNT(*) FROM "user") as users, (SELECT COUNT(*) FROM tributary) as tributaries, (SELECT COUNT(*) FROM usertributary) as memberships,