import { fileURLToPath as __fileURLToPath } from 'url';
import { dirname as __pathDirname } from 'path';
const __filename = __fileURLToPath(import.meta.url);
const __dirname = __pathDirname(__filename);
import { createClient } from "redis";
import {
  S3Client,
  PutObjectCommand,
  GetObjectCommand,
  DeleteObjectCommand
} from "@aws-sdk/client-s3";
import { Storage } from "@google-cloud/storage";
import { logger } from "../monitoring/logger.js";
import * as zlib from "zlib";
import { promisify } from "util";
function getEnv(key, defaultValue) {
  const value = process.env[key];
  if (value === void 0) {
    if (defaultValue !== void 0) return defaultValue;
    throw new Error(`Environment variable ${key} is required`);
  }
  return value;
}
function getOptionalEnv(key) {
  return process.env[key];
}
const gzipAsync = promisify(zlib.gzip);
const gunzipAsync = promisify(zlib.gunzip);
var StorageTier = /* @__PURE__ */ ((StorageTier2) => {
  StorageTier2["HOT"] = "hot";
  StorageTier2["WARM"] = "warm";
  StorageTier2["COLD"] = "cold";
  return StorageTier2;
})(StorageTier || {});
const DEFAULT_RAILWAY_CONFIG = {
  redis: {
    url: process.env["REDIS_URL"] || "redis://localhost:6379",
    ttlSeconds: 86400,
    // 24 hours
    maxMemoryMb: 100
    // 100MB Redis limit
  },
  railwayBuckets: {
    endpoint: process.env["RAILWAY_BUCKET_ENDPOINT"] || "https://buckets.railway.app",
    bucket: process.env["RAILWAY_BUCKET_NAME"] || "stackmemory-warm",
    accessKeyId: process.env["RAILWAY_BUCKET_ACCESS_KEY"] || "",
    secretAccessKey: process.env["RAILWAY_BUCKET_SECRET_KEY"] || "",
    region: "us-east-1"
  },
  gcs: {
    bucketName: process.env["GCS_BUCKET"] || "stackmemory-cold",
    projectId: process.env["GCP_PROJECT_ID"] || "stackmemory",
    keyFilename: process.env["GCP_KEY_FILE"]
  },
  tiers: {
    hotHours: 24,
    warmDays: 30,
    compressionScore: 0.4
  }
};
class RailwayOptimizedStorage {
  redisClient;
  railwayS3;
  gcsStorage;
  localDb;
  config;
  configManager;
  metricsCache = /* @__PURE__ */ new Map();
  initialized;
  constructor(localDb, configManager, config) {
    this.localDb = localDb;
    this.configManager = configManager;
    this.config = { ...DEFAULT_RAILWAY_CONFIG, ...config };
    this.initializeSchema();
    this.initialized = this.initializeClients();
  }
  /**
   * Initialize storage clients
   */
  async initializeClients() {
    if (this.config.redis.url) {
      try {
        this.redisClient = createClient({ url: this.config.redis.url });
        this.redisClient.on("error", (err) => {
          logger.error("Redis client error", err);
        });
        await this.redisClient.connect();
        await this.redisClient.configSet("maxmemory-policy", "allkeys-lru");
        logger.info("Redis connected for hot tier storage");
      } catch (error) {
        logger.warn(
          "Redis connection failed, falling back to SQLite only",
          error
        );
      }
    }
    if (this.config.railwayBuckets.accessKeyId) {
      this.railwayS3 = new S3Client({
        endpoint: this.config.railwayBuckets.endpoint,
        region: this.config.railwayBuckets.region,
        credentials: {
          accessKeyId: this.config.railwayBuckets.accessKeyId,
          secretAccessKey: this.config.railwayBuckets.secretAccessKey
        },
        forcePathStyle: true
        // Required for Railway buckets
      });
      logger.info("Railway Buckets configured for warm tier");
    }
    if (this.config.gcs.projectId) {
      try {
        this.gcsStorage = new Storage({
          projectId: this.config.gcs.projectId,
          keyFilename: this.config.gcs.keyFilename
        });
        logger.info("GCS configured for cold tier storage");
      } catch (error) {
        logger.warn("GCS setup failed, will use Railway buckets only", error);
      }
    }
  }
  /**
   * Initialize database schema for tracking
   */
  initializeSchema() {
    this.localDb.exec(`
      CREATE TABLE IF NOT EXISTS storage_tiers (
        trace_id TEXT PRIMARY KEY,
        tier TEXT NOT NULL,
        location TEXT NOT NULL,
        original_size INTEGER,
        compressed_size INTEGER,
        compression_ratio REAL,
        access_count INTEGER DEFAULT 0,
        last_accessed INTEGER,
        created_at INTEGER,
        migrated_at INTEGER,
        score REAL,
        FOREIGN KEY (trace_id) REFERENCES traces(id) ON DELETE CASCADE
      )
    `);
    this.localDb.exec(`
      CREATE INDEX IF NOT EXISTS idx_storage_tier ON storage_tiers(tier);
      CREATE INDEX IF NOT EXISTS idx_storage_created ON storage_tiers(created_at);
      CREATE INDEX IF NOT EXISTS idx_storage_accessed ON storage_tiers(last_accessed);
    `);
  }
  /**
   * Store a trace in the appropriate tier
   */
  async storeTrace(trace) {
    await this.initialized;
    const score = trace.score;
    const age = Date.now() - trace.metadata.startTime;
    const ageHours = age / (1e3 * 60 * 60);
    let tier;
    if (ageHours < this.config.tiers.hotHours && score > this.config.tiers.compressionScore) {
      tier = "hot" /* HOT */;
    } else if (ageHours < this.config.tiers.warmDays * 24) {
      tier = "warm" /* WARM */;
    } else {
      tier = "cold" /* COLD */;
    }
    switch (tier) {
      case "hot" /* HOT */:
        await this.storeInRedis(trace);
        break;
      case "warm" /* WARM */:
        await this.storeInRailwayBuckets(trace);
        break;
      case "cold" /* COLD */:
        await this.storeInGCS(trace);
        break;
    }
    this.trackStorage(trace.id, tier, trace);
    return tier;
  }
  /**
   * Store trace in Redis (hot tier)
   */
  async storeInRedis(trace) {
    if (!this.redisClient) {
      return;
    }
    try {
      const key = `trace:${trace.id}`;
      const data = JSON.stringify(trace);
      let storedData;
      if (data.length > 1e4) {
        const compressed = await gzipAsync(data);
        storedData = compressed.toString("base64");
        await this.redisClient.hSet(key, {
          data: storedData,
          compressed: "true",
          score: trace.score.toString(),
          type: trace.type,
          timestamp: trace.metadata.startTime.toString()
        });
      } else {
        storedData = data;
        await this.redisClient.hSet(key, {
          data: storedData,
          compressed: "false",
          score: trace.score.toString(),
          type: trace.type,
          timestamp: trace.metadata.startTime.toString()
        });
      }
      await this.redisClient.expire(key, this.config.redis.ttlSeconds);
      await this.redisClient.zAdd("traces:by_score", {
        score: trace.score,
        value: trace.id
      });
      await this.redisClient.zAdd("traces:by_time", {
        score: trace.metadata.startTime,
        value: trace.id
      });
      logger.debug("Stored trace in Redis", {
        traceId: trace.id,
        size: data.length,
        compressed: data.length > 1e4
      });
    } catch (error) {
      logger.error("Failed to store in Redis", error);
      throw error;
    }
  }
  /**
   * Store trace in Railway Buckets (warm tier)
   */
  async storeInRailwayBuckets(trace) {
    if (!this.railwayS3) {
      throw new Error("Railway Buckets not configured");
    }
    try {
      const data = JSON.stringify(trace);
      const compressed = await gzipAsync(data);
      const date = new Date(trace.metadata.startTime);
      const key = `traces/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}/${String(date.getDate()).padStart(2, "0")}/${trace.id}.json.gz`;
      const command = new PutObjectCommand({
        Bucket: this.config.railwayBuckets.bucket,
        Key: key,
        Body: compressed,
        ContentType: "application/gzip",
        Metadata: {
          "trace-id": trace.id,
          "trace-type": trace.type,
          "trace-score": trace.score.toString(),
          "original-size": data.length.toString(),
          "compressed-size": compressed.length.toString()
        }
      });
      await this.railwayS3.send(command);
      if (this.redisClient) {
        await this.redisClient.del(`trace:${trace.id}`);
      }
      logger.info("Stored trace in Railway Buckets", {
        traceId: trace.id,
        key,
        originalSize: data.length,
        compressedSize: compressed.length,
        compressionRatio: (1 - compressed.length / data.length).toFixed(2)
      });
    } catch (error) {
      logger.error("Failed to store in Railway Buckets", error);
      throw error;
    }
  }
  /**
   * Store trace in GCS (cold tier)
   */
  async storeInGCS(trace) {
    if (!this.gcsStorage) {
      return this.storeInRailwayBuckets(trace);
    }
    try {
      const minimal = this.createMinimalTrace(trace);
      const data = JSON.stringify(minimal);
      const compressed = await gzipAsync(data);
      const date = new Date(trace.metadata.startTime);
      const key = `archive/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}/${trace.id}.json.gz`;
      const bucket = this.gcsStorage.bucket(this.config.gcs.bucketName);
      const file = bucket.file(key);
      await file.save(compressed, {
        metadata: {
          contentType: "application/gzip",
          metadata: {
            traceId: trace.id,
            traceType: trace.type,
            score: trace.score.toString(),
            originalTools: trace.tools.length.toString()
          }
        },
        storageClass: "COLDLINE"
        // Use Coldline for cost optimization
      });
      if (this.railwayS3) {
        try {
          const warmKey = this.getWarmTierKey(trace);
          await this.railwayS3.send(
            new DeleteObjectCommand({
              Bucket: this.config.railwayBuckets.bucket,
              Key: warmKey
            })
          );
        } catch (error) {
        }
      }
      logger.info("Archived trace to GCS", {
        traceId: trace.id,
        key,
        originalSize: JSON.stringify(trace).length,
        compressedSize: compressed.length
      });
    } catch (error) {
      logger.error("Failed to store in GCS", error);
      throw error;
    }
  }
  /**
   * Create minimal trace for cold storage
   */
  createMinimalTrace(trace) {
    return {
      id: trace.id,
      type: trace.type,
      score: trace.score,
      summary: trace.summary,
      metadata: {
        startTime: trace.metadata.startTime,
        endTime: trace.metadata.endTime,
        filesModified: trace.metadata.filesModified.length,
        errorsCount: trace.metadata.errorsEncountered.length,
        decisionsCount: trace.metadata.decisionsRecorded.length,
        causalChain: trace.metadata.causalChain
      },
      toolSummary: {
        count: trace.tools.length,
        types: [...new Set(trace.tools.map((t) => t.tool))],
        firstTool: trace.tools[0]?.tool,
        lastTool: trace.tools[trace.tools.length - 1]?.tool
      },
      compressed: trace.compressed
    };
  }
  /**
   * Retrieve a trace from any tier
   */
  async retrieveTrace(traceId) {
    await this.initialized;
    const location = this.localDb.prepare("SELECT tier, location FROM storage_tiers WHERE trace_id = ?").get(traceId);
    if (!location) {
      return null;
    }
    this.localDb.prepare(
      "UPDATE storage_tiers SET access_count = access_count + 1, last_accessed = ? WHERE trace_id = ?"
    ).run(Date.now(), traceId);
    switch (location.tier) {
      case "hot" /* HOT */:
        return this.retrieveFromRedis(traceId);
      case "warm" /* WARM */:
        return this.retrieveFromRailwayBuckets(traceId, location.location);
      case "cold" /* COLD */:
        return this.retrieveFromGCS(traceId, location.location);
      default:
        return null;
    }
  }
  /**
   * Retrieve from Redis
   */
  async retrieveFromRedis(traceId) {
    if (!this.redisClient) return null;
    try {
      const key = `trace:${traceId}`;
      const data = await this.redisClient.hGetAll(key);
      if (!data || !data.data) return null;
      let traceData;
      if (data.compressed === "true") {
        const compressed = Buffer.from(data.data, "base64");
        const decompressed = await gunzipAsync(compressed);
        traceData = decompressed.toString();
      } else {
        traceData = data.data;
      }
      return JSON.parse(traceData);
    } catch (error) {
      logger.error("Failed to retrieve from Redis", error);
      return null;
    }
  }
  /**
   * Retrieve from Railway Buckets
   */
  async retrieveFromRailwayBuckets(traceId, key) {
    if (!this.railwayS3) return null;
    try {
      const command = new GetObjectCommand({
        Bucket: this.config.railwayBuckets.bucket,
        Key: key
      });
      const response = await this.railwayS3.send(command);
      const compressed = await response.Body?.transformToByteArray();
      if (!compressed) return null;
      const decompressed = await gunzipAsync(Buffer.from(compressed));
      return JSON.parse(decompressed.toString());
    } catch (error) {
      logger.error("Failed to retrieve from Railway Buckets", error);
      return null;
    }
  }
  /**
   * Retrieve from GCS
   */
  async retrieveFromGCS(traceId, key) {
    if (!this.gcsStorage) return null;
    try {
      const bucket = this.gcsStorage.bucket(this.config.gcs.bucketName);
      const file = bucket.file(key);
      const [compressed] = await file.download();
      const decompressed = await gunzipAsync(compressed);
      return JSON.parse(decompressed.toString());
    } catch (error) {
      logger.error("Failed to retrieve from GCS", error);
      return null;
    }
  }
  /**
   * Track storage in database
   */
  trackStorage(traceId, tier, trace) {
    const originalSize = JSON.stringify(trace).length;
    const compressedSize = Math.floor(originalSize * 0.3);
    this.localDb.prepare(
      `
      INSERT OR REPLACE INTO storage_tiers (
        trace_id, tier, location, original_size, compressed_size,
        compression_ratio, access_count, last_accessed, created_at,
        migrated_at, score
      ) VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)
    `
    ).run(
      traceId,
      tier,
      this.getStorageLocation(trace, tier),
      originalSize,
      compressedSize,
      1 - compressedSize / originalSize,
      Date.now(),
      trace.metadata.startTime,
      Date.now(),
      trace.score
    );
  }
  /**
   * Get storage location key
   */
  getStorageLocation(trace, tier) {
    const date = new Date(trace.metadata.startTime);
    const year = date.getFullYear();
    const month = String(date.getMonth() + 1).padStart(2, "0");
    const day = String(date.getDate()).padStart(2, "0");
    switch (tier) {
      case "hot" /* HOT */:
        return `redis:trace:${trace.id}`;
      case "warm" /* WARM */:
        return `traces/${year}/${month}/${day}/${trace.id}.json.gz`;
      case "cold" /* COLD */:
        return `archive/${year}/${month}/${trace.id}.json.gz`;
    }
  }
  /**
   * Get warm tier key for a trace
   */
  getWarmTierKey(trace) {
    const date = new Date(trace.metadata.startTime);
    return `traces/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}/${String(date.getDate()).padStart(2, "0")}/${trace.id}.json.gz`;
  }
  /**
   * Migrate traces between tiers based on age
   */
  async migrateTiers() {
    const results = {
      hotToWarm: 0,
      warmToCold: 0,
      errors: []
    };
    const now = Date.now();
    const candidates = this.localDb.prepare(
      `
      SELECT trace_id, tier, created_at, score
      FROM storage_tiers
      WHERE tier != 'cold'
      ORDER BY created_at ASC
    `
    ).all();
    for (const candidate of candidates) {
      const ageHours = (now - candidate.created_at) / (1e3 * 60 * 60);
      try {
        if (candidate.tier === "hot" /* HOT */ && ageHours > this.config.tiers.hotHours) {
          const trace = await this.retrieveTrace(candidate.trace_id);
          if (trace) {
            await this.storeInRailwayBuckets(trace);
            this.trackStorage(candidate.trace_id, "warm" /* WARM */, trace);
            results.hotToWarm++;
          }
        } else if (candidate.tier === "warm" /* WARM */ && ageHours > this.config.tiers.warmDays * 24) {
          const trace = await this.retrieveTrace(candidate.trace_id);
          if (trace) {
            await this.storeInGCS(trace);
            this.trackStorage(candidate.trace_id, "cold" /* COLD */, trace);
            results.warmToCold++;
          }
        }
      } catch (error) {
        results.errors.push(
          `Failed to migrate ${candidate.trace_id}: ${error}`
        );
      }
    }
    logger.info("Tier migration completed", results);
    return results;
  }
  /**
   * Get storage statistics
   */
  getStorageStats() {
    const tierStats = this.localDb.prepare(
      `
      SELECT 
        tier,
        COUNT(*) as count,
        SUM(original_size) as total_original,
        SUM(compressed_size) as total_compressed,
        AVG(compression_ratio) as avg_compression,
        AVG(access_count) as avg_access
      FROM storage_tiers
      GROUP BY tier
    `
    ).all();
    const ageDistribution = this.localDb.prepare(
      `
      SELECT 
        CASE 
          WHEN (? - created_at) / 3600000 < 24 THEN '< 24h'
          WHEN (? - created_at) / 86400000 < 7 THEN '1-7d'
          WHEN (? - created_at) / 86400000 < 30 THEN '7-30d'
          ELSE '30d+'
        END as age_group,
        COUNT(*) as count
      FROM storage_tiers
      GROUP BY age_group
    `
    ).all(Date.now(), Date.now(), Date.now());
    return {
      byTier: tierStats,
      byAge: ageDistribution,
      totalTraces: tierStats.reduce((sum, t) => sum + t.count, 0),
      totalSize: tierStats.reduce(
        (sum, t) => sum + t.total_original,
        0
      ),
      compressedSize: tierStats.reduce(
        (sum, t) => sum + t.total_compressed,
        0
      )
    };
  }
  /**
   * Clean up expired data
   */
  async cleanup() {
    let cleaned = 0;
    const cutoff = Date.now() - 90 * 24 * 60 * 60 * 1e3;
    const result = this.localDb.prepare(
      `
      DELETE FROM storage_tiers
      WHERE tier = 'cold' AND created_at < ? AND access_count = 0
    `
    ).run(cutoff);
    cleaned = result.changes;
    logger.info("Cleanup completed", { removed: cleaned });
    return cleaned;
  }
}
export {
  DEFAULT_RAILWAY_CONFIG,
  RailwayOptimizedStorage,
  StorageTier
};
//# sourceMappingURL=railway-optimized-storage.js.map
