Add Nairobi Information Collector application

nairobi-info-collector/.env.example

@@ -0,0 +1,88 @@
+# Application Settings
+APP_NAME="Nairobi Information Collector"
+APP_VERSION="1.0.0"
+DEBUG=True
+ENVIRONMENT=development
+
+# Server Configuration
+HOST=0.0.0.0
+PORT=8000
+
+# Database Configuration
+DATABASE_URL=postgresql://nairobiuser:password@localhost:5432/nairobi_info
+# For SQLite (development): sqlite:///./nairobi_info.db
+
+# Redis Configuration
+REDIS_URL=redis://localhost:6379/0
+REDIS_PASSWORD=
+
+# API Keys - News Sources
+NEWS_API_KEY=your_news_api_key_here
+
+# API Keys - Social Media
+TWITTER_API_KEY=your_twitter_api_key
+TWITTER_API_SECRET=your_twitter_api_secret
+TWITTER_ACCESS_TOKEN=your_twitter_access_token
+TWITTER_ACCESS_SECRET=your_twitter_access_secret
+TWITTER_BEARER_TOKEN=your_twitter_bearer_token
+
+INSTAGRAM_USERNAME=your_instagram_username
+INSTAGRAM_PASSWORD=your_instagram_password
+
+# API Keys - Maps & Location
+GOOGLE_MAPS_API_KEY=your_google_maps_api_key
+FOURSQUARE_API_KEY=your_foursquare_api_key
+
+# API Keys - NLP & AI
+OPENAI_API_KEY=your_openai_api_key
+ANTHROPIC_API_KEY=your_anthropic_api_key
+
+# Collection Settings
+COLLECTION_INTERVAL_SECONDS=300
+MAX_ITEMS_PER_SOURCE=100
+REQUEST_TIMEOUT_SECONDS=30
+MAX_RETRIES=3
+
+# Rate Limiting
+RATE_LIMIT_REQUESTS_PER_MINUTE=60
+RATE_LIMIT_REQUESTS_PER_HOUR=1000
+
+# Scraping Settings
+USER_AGENT="Mozilla/5.0 (compatible; NairobiInfoBot/1.0)"
+RESPECT_ROBOTS_TXT=True
+ENABLE_CACHING=True
+CACHE_TTL_SECONDS=3600
+
+# Data Processing
+ENABLE_NLP_PROCESSING=True
+ENABLE_SENTIMENT_ANALYSIS=True
+ENABLE_AUTO_CATEGORIZATION=True
+MIN_RELIABILITY_SCORE=0.5
+
+# Logging
+LOG_LEVEL=INFO
+LOG_FILE=logs/nairobi_collector.log
+
+# Security
+SECRET_KEY=your-secret-key-change-this-in-production
+API_KEY_HEADER=X-API-Key
+ALLOWED_ORIGINS=http://localhost:3000,http://localhost:8000
+
+# Monitoring
+SENTRY_DSN=
+ENABLE_METRICS=True
+METRICS_PORT=9090
+
+# Feature Flags
+ENABLE_SOCIAL_MEDIA_COLLECTION=True
+ENABLE_NEWS_COLLECTION=True
+ENABLE_GOVERNMENT_COLLECTION=True
+ENABLE_TOURISM_COLLECTION=True
+ENABLE_BUSINESS_COLLECTION=True
+
+# Email Notifications (for alerts)
+SMTP_HOST=smtp.gmail.com
+SMTP_PORT=587
+[email protected]
+SMTP_PASSWORD=your_app_password
+[email protected]

nairobi-info-collector/.gitignore

@@ -0,0 +1,65 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+env/
+ENV/
+.venv
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# Database
+*.db
+*.sqlite
+*.sqlite3
+
+# Logs
+logs/
+*.log
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Jupyter
+.ipynb_checkpoints
+
+# Docker
+*.pid
+.dockerignore
+
+# OS
+Thumbs.db

nairobi-info-collector/Dockerfile

@@ -0,0 +1,38 @@
+# Dockerfile for Nairobi Information Collector
+
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    postgresql-client \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Download spaCy model (for NLP)
+RUN python -m spacy download en_core_web_sm
+
+# Copy application code
+COPY . .
+
+# Create logs directory
+RUN mkdir -p logs
+
+# Expose port
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/api/v1/health || exit 1
+
+# Run the application
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

nairobi-info-collector/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Nairobi Information Collector
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.