"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const utils_1 = require("../../src/utils");
const TokenCompressor_1 = require("../../src/processors/TokenCompressor");
const services_1 = require("../../src/services");
describe("TokenCompressor - Large File Test", () => {
    let tokenCompressor;
    const bigstringPath = "tests/compressor/bigstring.txt";
    const jsonPath = "tests/compressor/githubjson.txt";
    beforeAll(() => {
        const { Tools } = (0, services_1.services)();
        tokenCompressor = new TokenCompressor_1.TokenCompressor(Tools);
    });
    afterEach(() => {
        tokenCompressor.clearStorage();
    });
    test("should compress large file contents and allow retrieval via chunks", async () => {
        const fileBuffer = await (0, utils_1.readFile)(bigstringPath);
        const fileContents = fileBuffer.toString();
        console.log(`Original file size: ${fileContents.length} characters`);
        console.log(`Estimated tokens: ${Math.ceil(fileContents.length / 4)}`);
        const compressed = tokenCompressor.compressContent(fileContents, bigstringPath);
        console.log(`Compressed result: ${compressed}`);
        expect(compressed).toContain("[COMPRESSED_STRING");
        expect(compressed).toContain("Key:");
        expect(compressed).toContain("chunks]");
        expect(compressed.length).toBeLessThan(fileContents.length);
        const keyMatch = compressed.match(/Key: (compressed_[a-z0-9_]+)/);
        expect(keyMatch).not.toBeNull();
        const firstKey = keyMatch[1];
        const firstChunk = tokenCompressor.retrieveString(firstKey);
        expect(firstChunk).toBeTruthy();
        expect(firstChunk.length).toBeGreaterThan(0);
        expect(fileContents.startsWith(firstChunk.split("[NEXT_CHUNK_KEY:")[0]));
        const currentChunk = firstChunk;
        let reconstructed = "";
        let chunkCount = 0;
        const maxChunks = 100;
        while (currentChunk && chunkCount < maxChunks) {
            chunkCount++;
            const nextKeyMatch = currentChunk.match(/\[NEXT_CHUNK_KEY: ([^\]]+)\]/);
            if (nextKeyMatch) {
                const nextKey = nextKeyMatch[1];
                const retrieved = await tokenCompressor.retrieveString(nextKey);
                console.log(`Retrieved chunk ${chunkCount} with key: ${nextKey}, length: ${retrieved.length}`);
            }
            else {
                reconstructed += currentChunk;
                break;
            }
        }
        console.log(`Retrieved ${chunkCount} chunks`);
        console.log(`Reconstructed size: ${reconstructed.length} characters`);
        expect(chunkCount).toBeGreaterThan(1);
    });
    test("should handle compression threshold correctly", async () => {
        const fileBuffer = await (0, utils_1.readFile)(bigstringPath);
        const fileContents = fileBuffer.toString();
        const estimatedTokens = Math.ceil(fileContents.length / 4);
        expect(estimatedTokens).toBeGreaterThan(4000);
        const compressed = tokenCompressor.compressContent(fileContents);
        expect(compressed).toContain("[COMPRESSED_STRING");
        const compressionRatio = compressed.length / fileContents.length;
        expect(compressionRatio).toBeLessThan(0.01);
    });
    test("should handle json compression", async () => {
        const fileBuffer = await (0, utils_1.readFile)(jsonPath);
        const fileContents = fileBuffer.toString();
        const estimatedTokens = Math.ceil(fileContents.length / 4);
        expect(estimatedTokens).toBeGreaterThan(4000);
        const compressed = tokenCompressor.compressContent(fileContents);
        console.log({ compressedJson: compressed });
        const compressedObj = JSON.parse(compressed);
        expect(compressedObj._mcp_format).toBe(true);
        expect(compressedObj._data).toBeDefined();
        expect(compressedObj._schema_key).toBeDefined();
        const compressionRatio = compressed.length / fileContents.length;
        expect(compressionRatio).toBeLessThan(0.5);
    });
    test("should detect json", async () => {
        const fileBuffer = await (0, utils_1.readFile)(jsonPath);
        const fileContents = fileBuffer.toString();
        const compressed = tokenCompressor.tryParseJson(fileContents);
        expect(compressed).toBeTruthy();
    });
    test("should analyze githubjson.txt compression with key chain utilities", async () => {
        const fileBuffer = await (0, utils_1.readFile)(jsonPath);
        const fileContents = fileBuffer.toString();
        console.log(`\n=== GitHub JSON Compression Analysis ===`);
        console.log(`Original file size: ${fileContents.length} characters`);
        console.log(`Estimated tokens: ${Math.ceil(fileContents.length / 4)}`);
        const compressed = tokenCompressor.compressContent(fileContents);
        console.log(`\nCompressed representation length: ${compressed.length} characters`);
        console.log(`\nCompressed output:\n${compressed.substring(0, 500)}...\n`);
        const rootKeys = tokenCompressor.extractKeys(compressed);
        console.log(`\nRoot keys found: ${rootKeys.length}`);
        console.log(`Keys: ${rootKeys.join(", ")}\n`);
        expect(rootKeys.length).toBeGreaterThan(0);
        for (const rootKey of rootKeys) {
            console.log(`\n--- Analyzing key chain for: ${rootKey} ---`);
            const keyChain = tokenCompressor.getKeyChain(rootKey);
            console.log(`Total keys in chain: ${keyChain.length}`);
            const byDepth = keyChain.reduce((acc, info) => {
                if (!acc[info.depth]) {
                    acc[info.depth] = [];
                }
                acc[info.depth].push(info);
                return acc;
            }, {});
            console.log(`\nKeys by depth:`);
            for (const [depth, infos] of Object.entries(byDepth)) {
                console.log(`  Depth ${depth}: ${infos.length} keys`);
                for (const info of infos) {
                    console.log(`    - ${info.key}:`);
                    console.log(`        Size: ${info.size} chars (${info.tokens} tokens)`);
                    console.log(`        Type: ${info.type}`);
                    console.log(`        Child keys: ${info.childKeys.length}`);
                    if (info.nextChunkKey) {
                        console.log(`        Next chunk: ${info.nextChunkKey}`);
                    }
                }
            }
            for (const info of keyChain) {
                const data = await tokenCompressor.retrieveString(info.key);
                console.log({ data });
                if (info.type === "child") {
                    expect(info.tokens).toBeGreaterThan(1000);
                }
                expect(info.tokens).toBeLessThan(tokenCompressor.maxTokens * 1.5);
            }
            const totalSize = keyChain.reduce((sum, info) => sum + info.size, 0);
            const totalTokens = keyChain.reduce((sum, info) => sum + info.tokens, 0);
            console.log(`\nTotal stored size: ${totalSize} characters (${totalTokens} tokens)`);
            console.log(`Storage efficiency: Stored ${keyChain.length} chunks for original content`);
        }
        console.log(`\n=== Storage State ===`);
        console.log(`Total keys in storage: ${tokenCompressor.getStorageSize()}`);
        console.log(`All storage keys: ${tokenCompressor.getStorageKeys().join(", ")}`);
        expect(tokenCompressor.getStorageSize()).toBeGreaterThan(0);
        const parsedCompressed = tokenCompressor.tryParseJson(compressed);
        expect(parsedCompressed).toBeTruthy();
        let dataKeys = [];
        if (parsedCompressed && typeof parsedCompressed === "object") {
            const extractAllKeys = (obj) => {
                const keys = [];
                const str = JSON.stringify(obj);
                keys.push(...tokenCompressor.extractKeys(str));
                return keys;
            };
            dataKeys = extractAllKeys(parsedCompressed);
        }
        console.log(`\nKeys found in compressed structure: ${dataKeys.length}`);
        const storedContent = dataKeys.length > 0 ? tokenCompressor.retrieveString(dataKeys[0]) : null;
        const embeddedKeys = tokenCompressor.extractKeys(storedContent);
        console.log(`\nKeys embedded in first stored chunk: ${embeddedKeys.length}`);
        if (embeddedKeys.length > 0) {
            console.log(`Embedded keys: ${embeddedKeys.join(", ")}`);
        }
    });
    test("should demonstrate improved compression with schema and low-signal detection", async () => {
        const fileBuffer = await (0, utils_1.readFile)(jsonPath);
        const fileContents = fileBuffer.toString();
        console.log(`\n=== Compression Performance Analysis ===`);
        console.log(`Original file size: ${fileContents.length} characters`);
        console.log(`Estimated tokens: ${Math.ceil(fileContents.length / 4)}`);
        const compressed = tokenCompressor.compressContent(fileContents);
        console.log(`\nCompressed representation: ${compressed.length} characters`);
        const compressionRatio = ((1 - compressed.length / fileContents.length) *
            100).toFixed(2);
        console.log(`Compression ratio: ${compressionRatio}%`);
        console.log(`\nCompressed output preview:\n${compressed.substring(0, 800)}`);
        const compressedObj = JSON.parse(compressed);
        console.log(`\n=== Schema Information ===`);
        if (compressedObj._schema_key) {
            console.log(`Schema key found: ${compressedObj._schema_key}`);
            const schema = tokenCompressor.retrieveString(compressedObj._schema_key);
            if (schema) {
                console.log(`Schema structure:`);
                console.log(JSON.stringify(schema, null, 2).substring(0, 1000));
            }
        }
        console.log(`\n=== Low-Signal Property Compression ===`);
        if (compressedObj._data && Array.isArray(compressedObj._data)) {
            const firstItem = compressedObj._data[0];
            if (typeof firstItem === "string" &&
                firstItem.includes("COMPRESSED_JSON")) {
                const keyMatch = firstItem.match(/Key: (compressed_[a-z0-9_]+)/);
                if (keyMatch) {
                    const itemKey = keyMatch[1];
                    const itemData = tokenCompressor.retrieveString(itemKey);
                    if (itemData) {
                        const item = JSON.parse(itemData);
                        const innerItem = JSON.parse(item.text);
                        if (Array.isArray(innerItem)) {
                            const sampleItem = innerItem[0];
                            if (sampleItem._compressed_properties_key) {
                                console.log(`Sample item has ${sampleItem._compressed_property_names.length} compressed properties`);
                                console.log(`Compressed properties: ${sampleItem._compressed_property_names
                                    .slice(0, 10)
                                    .join(", ")}...`);
                                console.log(`Info: ${sampleItem._compression_info}`);
                                const compressedProps = tokenCompressor.getCompressedProperties(sampleItem._compressed_properties_key);
                                if (compressedProps) {
                                    const propNames = Object.keys(compressedProps.compressed_properties);
                                    console.log(`\nRetrieved ${propNames.length} compressed properties successfully`);
                                    console.log(`Sample compressed property keys: ${propNames
                                        .slice(0, 5)
                                        .join(", ")}`);
                                }
                            }
                        }
                    }
                }
            }
        }
        console.log(`\n=== Storage Analysis ===`);
        const totalKeys = tokenCompressor.getStorageSize();
        console.log(`Total storage keys: ${totalKeys}`);
        let totalStoredSize = 0;
        for (const key of tokenCompressor.getStorageKeys()) {
            const data = tokenCompressor.retrieveString(key);
            if (data) {
                totalStoredSize += data.length;
            }
        }
        console.log(`Total stored data: ${totalStoredSize} characters`);
        console.log(`Compressed representation: ${compressed.length} characters`);
        console.log(`Storage overhead ratio: ${((totalStoredSize / fileContents.length) *
            100).toFixed(2)}%`);
        console.log(`Initial view compression: ${compressionRatio}%`);
        expect(parseFloat(compressionRatio)).toBeGreaterThan(60);
        if (compressedObj._schema_key) {
            const schema = tokenCompressor.retrieveString(compressedObj._schema_key);
            expect(schema).toBeTruthy();
        }
        expect(totalKeys).toBeGreaterThan(5);
        console.log(`\n✓ Compression improvements verified!`);
    });
    test("should not double-compress already compressed content", async () => {
        const fileBuffer = await (0, utils_1.readFile)(jsonPath);
        const fileContents = fileBuffer.toString();
        console.log(`\n=== Testing Double Compression Prevention ===`);
        console.log(`Original file size: ${fileContents.length} characters`);
        const message = JSON.parse(fileContents);
        const messageToCompress = JSON.parse(JSON.stringify(message));
        await tokenCompressor.compressMessage(messageToCompress);
        const firstCompression = messageToCompress.content[0].text;
        console.log(`\nAfter first compression: ${firstCompression.length} characters`);
        console.log(`First compression preview:\n${firstCompression.substring(0, 300)}...`);
        const firstParsed = tokenCompressor.tryParseJson(firstCompression);
        expect(firstParsed).toBeTruthy();
        expect(firstParsed._schema_key).toBeDefined();
        expect(firstParsed.data).toBeDefined();
        await tokenCompressor.compressMessage(messageToCompress);
        const secondCompression = messageToCompress.content[0].text;
        console.log(`\nAfter second compression: ${secondCompression.length} characters`);
        console.log(`Second compression preview:\n${secondCompression.substring(0, 300)}...`);
        const secondParsed = tokenCompressor.tryParseJson(secondCompression);
        expect(secondParsed).toBeTruthy();
        console.log(`\nSecond parsed structure keys:`, Object.keys(secondParsed));
        console.log(`Full structure:`, JSON.stringify(secondParsed, null, 2).substring(0, 500));
        const isOverCompressed = Object.keys(secondParsed).length === 3 &&
            secondParsed._mcp_format === true &&
            secondParsed._raw_structure !== undefined &&
            secondParsed._schema_key !== undefined;
        console.log(`\nIs over-compressed: ${isOverCompressed}`);
        expect(secondCompression).toBe(firstCompression);
        expect(secondParsed).toEqual(firstParsed);
        expect(secondParsed.data).toBeDefined();
        expect(Array.isArray(secondParsed.data)).toBe(true);
        expect(isOverCompressed).toBe(false);
        console.log(`\n✓ Double compression prevention verified!`);
    });
});
//# sourceMappingURL=bigstring.test.js.map