create-typesense-collections.ts

Field	Value
Type	TypeScript
Source	Forge/Typesense/Maintenance/create-typesense-collections.ts
Parent	Forge
GitHub	Forge/Typesense/Maintenance/create-typesense-collections.ts

This page is auto-generated. Edit the source to change the content.

/**
 * create-typesense-collections.ts — Create Typesense collections for Uvilo OS
 *
 * Creates the `uvilo` and `uvilo_docs` collections in Typesense.
 * If a collection already exists, it is deleted and recreated.
 *
 * Collections:
 *   uvilo       — Repo documents (indexed by Forge/Typesense/Maintenance/index-department)
 *                 and website pages (indexed by .internal/scrape-site.py)
 *   uvilo_docs  — Website search documents (indexed by DocSearch scraper
 *                 via .github/workflows/index-search.yml). Created here as
 *                 a placeholder; the scraper will set its own schema on
 *                 first run, but pre-creating ensures it doesn't fail on a
 *                 fresh Typesense instance.
 *
 * Usage:
 *   TYPESENSE_URL=https://typesense-production-xxx.up.railway.app \
 *   TYPESENSE_ADMIN_KEY=xxx \
 *   npx tsx Forge/Typesense/Maintenance/create-typesense-collections.ts
 *
 * On Railway (from forge-bash):
 *   eval $(cat /proc/1/environ | tr '\0' '\n' | grep TYPESENSE)
 *   npx tsx /workspace/erik/uvilo-os/Forge/Typesense/Maintenance/create-typesense-collections.ts
 */

// ---------------------------------------------------------------------------
// Env var reading — matches .internal/index-department.py pattern
// ---------------------------------------------------------------------------

function getEnv(key: string): string {
  // Try /proc/1/environ first (Railway container)
  try {
    const fs = require("fs");
    const env = fs.readFileSync("/proc/1/environ", "utf-8");
    for (const entry of env.split("\0")) {
      if (entry.startsWith(`${key}=`)) {
        return entry.split("=").slice(1).join("=");
      }
    }
  } catch {
    // Not on Railway, fall through
  }
  const val = process.env[key];
  if (!val) {
    console.error(`ERROR: ${key} is not set. Set it in the environment or via /proc/1/environ.`);
    process.exit(1);
  }
  return val;
}

const TYPESENSE_URL = getEnv("TYPESENSE_URL");
const TYPESENSE_ADMIN_KEY = getEnv("TYPESENSE_ADMIN_KEY");

// ---------------------------------------------------------------------------
// Collection schemas
// ---------------------------------------------------------------------------

const UVILO_SCHEMA = {
  name: "uvilo",
  fields: [
    { name: "id", type: "string" },
    { name: "title", type: "string" },
    { name: "summary", type: "string" },
    { name: "content", type: "string" },
    { name: "path", type: "string" },
    { name: "department", type: "string", facet: true },
    { name: "project", type: "string", facet: true },
    { name: "type", type: "string", facet: true },
    { name: "source", type: "string", facet: true },
    { name: "status", type: "string", facet: true },
    { name: "visibility", type: "string", facet: true },
    { name: "owner", type: "string", facet: true },
    {
      name: "embedding",
      type: "float[]",
      num_dim: 1536,
      optional: true,
      // Embeddings are computed client-side using OpenAI text-embedding-3-small.
      // Server-side embed config was attempted but Typesense couldn't reach OpenAI from Railway.
    },
  ],
  default_sorting_field: "",
};

/**
 * Minimal schema for uvilo_docs. The DocSearch scraper (typesense/docsearch-scraper)
 * will overwrite this with its full schema on first run, but the collection must
 * exist before the scraper can write to it.
 */
const UVILO_DOCS_SCHEMA = {
  name: "uvilo_docs",
  fields: [
    { name: "url", type: "string" },
    { name: "title", type: "string" },
    { name: "hierarchy.lvl0", type: "string", facet: true, optional: true },
    { name: "hierarchy.lvl1", type: "string", facet: true, optional: true },
    { name: "hierarchy.lvl2", type: "string", facet: true, optional: true },
    { name: "hierarchy.lvl3", type: "string", facet: true, optional: true },
    { name: "hierarchy.lvl4", type: "string", facet: true, optional: true },
    { name: "hierarchy.lvl5", type: "string", facet: true, optional: true },
    { name: "content", type: "string", optional: true },
    { name: "type", type: "string", facet: true },
    { name: "objectID", type: "string" },
  ],
  default_sorting_field: "",
};

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

const headers = {
  "X-TYPESENSE-API-KEY": TYPESENSE_ADMIN_KEY,
  "Content-Type": "application/json",
};

async function deleteCollection(name: string): Promise<void> {
  const res = await fetch(`${TYPESENSE_URL}/collections/${name}`, {
    method: "DELETE",
    headers,
  });
  if (res.ok) {
    console.log(`  Deleted existing '${name}' collection.`);
  } else if (res.status === 404) {
    console.log(`  Collection '${name}' does not exist — creating fresh.`);
  } else {
    const err = await res.text();
    console.error(`  Warning: failed to delete '${name}': ${err}`);
  }
}

async function createCollection(schema: object): Promise<void> {
  const name = (schema as any).name;

  // Delete first if it exists
  await deleteCollection(name);

  // Create
  const res = await fetch(`${TYPESENSE_URL}/collections`, {
    method: "POST",
    headers,
    body: JSON.stringify(schema),
  });

  if (!res.ok) {
    const err = await res.text();
    console.error(`  Failed to create '${name}': ${err}`);
    process.exit(1);
  }

  const result = await res.json();
  const fields = result.fields.map((f: any) => f.name).join(", ");
  console.log(`  Created '${name}' — fields: ${fields}`);
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

async function main() {
  console.log("Creating Typesense collections...\n");
  console.log(`  TYPESENSE_URL: ${TYPESENSE_URL}\n`);

  await createCollection(UVILO_SCHEMA);
  await createCollection(UVILO_DOCS_SCHEMA);

  console.log("\nDone. Next steps:");
  console.log("  1. cd Forge/Typesense/Maintenance/index-department && npm run build");
  console.log("  2. node Forge/Typesense/Maintenance/index-department/dist/index.js --all");
  console.log("  3. Run scrape-site.py to populate 'uvilo' with website pages");
  console.log("  4. Push to dev to trigger the index-search workflow (populates 'uvilo_docs')");
}

main().catch((err) => {
  console.error("Fatal error:", err);
  process.exit(1);
});