This commit is contained in:
Nikita
2024-11-17 21:44:42 +01:00
parent 3b4cf7e415
commit 06f84cceed
17 changed files with 3 additions and 958 deletions

View File

@@ -1,88 +0,0 @@
// @ts-nocheck
async function devSeed() {
const { worker } = await startWorker({
accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
accountSecret: JAZZ_WORKER_SECRET
})
const user = (await (
await LaAccount.createAs(worker, {
creationProps: { name: "nikiv" }
})
).ensureLoaded({ root: { personalLinks: [], pages: [], todos: [] } }))!
const globalLinksGroup = Group.create({ owner: worker })
globalLinksGroup.addMember("everyone", "reader")
const globalLink1 = GlobalLink.create({ url: "https://google.com" }, { owner: globalLinksGroup })
const globalLink2 = GlobalLink.create({ url: "https://jazz.tools" }, { owner: globalLinksGroup })
// TODO: make note: optional
const personalLink1 = PersonalLink.create(
{ globalLink: globalLink1, type: "personalLink", note: "" },
{ owner: user }
)
const personalLink2 = PersonalLink.create(
{ globalLink: globalLink2, type: "personalLink", note: "Great framework" },
{ owner: user }
)
user.root.personalLinks.push(personalLink1)
user.root.personalLinks.push(personalLink2)
const pageOneTitle = "Physics"
const pageTwoTitle = "Karabiner"
const page1 = PersonalPage.create(
{ title: pageOneTitle, slug: generateUniqueSlug(pageOneTitle), content: "Physics is great" },
{ owner: user }
)
const page2 = PersonalPage.create(
{ title: pageTwoTitle, slug: generateUniqueSlug(pageTwoTitle), content: "Karabiner is great" },
{ owner: user }
)
user.root.personalPages?.push(page1)
user.root.personalPages?.push(page2)
const page1 = Page.create({ title: "Physics", content: "Physics is great" }, { owner: user })
const page2 = Page.create({ title: "Karabiner", content: "Karabiner is great" }, { owner: user })
user.root.pages.push(page1)
user.root.pages.push(page2)
const credentials = {
accountID: user.id,
accountSecret: (user._raw as RawControlledAccount).agentSecret
}
await Bun.write(
"./web/.env",
`VITE_SEED_ACCOUNTS='${JSON.stringify({
nikiv: credentials
})}'`
)
await Bun.write(
"./.env",
`VITE_SEED_ACCOUNTS='${JSON.stringify({
nikiv: credentials
})}'`
)
}
const globalLink = GlobalLink.create(
{
url: "https://google.com",
urlTitle: "Google",
protocol: "https"
},
{ owner: globalGroup }
)
const user = (await (
await LaAccount.createAs(worker, {
creationProps: { name: "nikiv" }
})
).ensureLoaded({ root: { personalLinks: [], pages: [], todos: [] } }))!
console.log(process.env.JAZZ_GLOBAL_GROUP!, "group")
console.log(worker)
// TODO: type err
console.log(globalGroup, "group")
return
const currentFilePath = import.meta.path
const connectionsFilePath = `${currentFilePath.replace("seed.ts", "/seed/connections.json")}`
const file = Bun.file(connectionsFilePath)
const fileContent = await file.text()
const topicsWithConnections = JSON.parse(fileContent)
// let topicsWithConnections = JSON.stringify(obj, null, 2)
console.log(topicsWithConnections)
// TODO: type err
topicsWithConnections.map(topic => {
const globalTopic = GlobalTopic.create({ name: topic.name, description: topic.description }, { owner: globalGroup })
})

View File

@@ -1,89 +0,0 @@
import { getEnvOrThrow } from "@/lib/utils"
import { PublicGlobalGroup } from "@/web/lib/schema/master/public-group"
import { startWorker } from "jazz-nodejs"
import { ID } from "jazz-tools"
const JAZZ_WORKER_SECRET = getEnvOrThrow("JAZZ_WORKER_SECRET")
async function run() {
try {
await readJazz()
} catch (err) {
console.log(err, "err")
}
}
async function readJazz() {
const { worker } = await startWorker({
accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
accountSecret: JAZZ_WORKER_SECRET
})
const globalGroupId = process.env.JAZZ_PUBLIC_GLOBAL_GROUP as ID<PublicGlobalGroup>
const globalGroup = await PublicGlobalGroup.load(globalGroupId, worker, {
root: {
topics: [
{
latestGlobalGuide: {
sections: [
{
links: [{}]
}
]
}
}
],
forceGraphs: [
{
connections: [{}]
}
]
}
})
if (!globalGroup) return // TODO: err
// wait 10 seconds
await new Promise(resolve => setTimeout(resolve, 10000))
/*
* Log forceGraphs
*/
const asJsonForceGraphs = globalGroup.root.forceGraphs.map(node => {
console.log({ node }, "node")
return {
name: node.name,
prettyName: node.prettyName,
connections: node.connections?.map(connection => {
return {
name: connection?.name
}
})
}
})
const asJson = globalGroup.root.topics?.map(node => {
return {
name: node.name,
prettyName: node.prettyName,
latestGlobalGuide: {
sections: node.latestGlobalGuide.sections.map(section => {
return {
title: section?.title,
links: section?.links?.map(link => {
return {
title: link?.title,
url: link?.url
}
})
}
})
}
}
})
console.log({ asJsonForceGraphs }, "asJsonForceGraphs")
console.log({ asJson }, "asJson")
}
await run()

View File

@@ -1,493 +0,0 @@
import { getEnvOrThrow } from "@/lib/utils"
import { Connection, ForceGraph, ListOfConnections, ListOfForceGraphs } from "@/web/lib/schema/master/force-graph"
import { PublicGlobalGroup, PublicGlobalGroupRoot } from "@/web/lib/schema/master/public-group"
import {
LatestGlobalGuide,
Link,
ListOfLinks,
ListOfSections,
ListOfTopics,
Section,
Topic
} from "@/web/lib/schema/master/topic"
import fs from "fs/promises"
import { startWorker } from "jazz-nodejs"
import { ID } from "jazz-tools"
import { appendFile } from "node:fs/promises"
import path from "path"
// Define interfaces for JSON data structures
interface LinkJson {
id?: ID<Link>
title: string
url: string
}
interface SectionJson {
title: string
links: LinkJson[]
}
interface TopicJson {
name: string
prettyName: string
latestGlobalGuide: {
sections: SectionJson[]
} | null
}
// Get the Jazz worker secret from environment variables
const JAZZ_WORKER_SECRET = getEnvOrThrow("JAZZ_WORKER_SECRET")
/**
* Manages links, handling deduplication and tracking duplicates.
*/
class LinkManager {
private links: Map<string, LinkJson> = new Map()
private duplicateCount: number = 0
/**
* Adds a link to the manager, tracking duplicates.
* @param link - The link to add.
*/
addLink(link: LinkJson) {
const key = link.url
if (this.links.has(key)) {
this.duplicateCount++
} else {
this.links.set(key, link)
}
}
/**
* Gets all unique links.
* @returns An array of unique links.
*/
getAllLinks() {
return Array.from(this.links.values())
}
/**
* Gets the count of duplicate links.
* @returns The number of duplicate links.
*/
getDuplicateCount() {
return this.duplicateCount
}
}
/**
* Starts a Jazz worker.
* @returns A Promise that resolves to the started worker.
*/
async function startJazzWorker() {
const { worker } = await startWorker({
accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
accountSecret: JAZZ_WORKER_SECRET
})
return worker
}
/**
* Sets up the global and admin groups.
*/
async function setup() {
console.log("Starting setup")
const worker = await startJazzWorker()
/*
* Create global group
*/
const publicGlobalGroup = PublicGlobalGroup.create({ owner: worker })
publicGlobalGroup.root = PublicGlobalGroupRoot.create(
{
topics: ListOfTopics.create([], { owner: publicGlobalGroup }),
forceGraphs: ListOfForceGraphs.create([], { owner: publicGlobalGroup })
},
{ owner: publicGlobalGroup }
)
publicGlobalGroup.addMember("everyone", "reader")
await appendFile("./.env", `\nJAZZ_PUBLIC_GLOBAL_GROUP=${JSON.stringify(publicGlobalGroup.id)}`)
/*
* Create admin group
*/
// const user = (await await LaAccount.createAs(worker, {
// creationProps: { name: "nikiv" }
// }))!
// const adminGlobalGroup = Group.create({ owner: worker })
// adminGlobalGroup.addMember(user, "admin")
// await appendFile("./.env", `\nJAZZ_ADMIN_GLOBAL_GROUP=${JSON.stringify(adminGlobalGroup.id)}`)
console.log("Setup completed successfully", publicGlobalGroup.id)
}
/**
* Loads the global group.
* @returns A Promise that resolves to the loaded global group.
* @throws Error if the global group fails to load.
*/
async function loadGlobalGroup() {
const worker = await startJazzWorker()
const globalGroupId = getEnvOrThrow("JAZZ_PUBLIC_GLOBAL_GROUP") as ID<PublicGlobalGroup>
const globalGroup = await PublicGlobalGroup.load(globalGroupId, worker, {
root: {
topics: [{ latestGlobalGuide: { sections: [] } }],
forceGraphs: [{ connections: [] }]
}
})
if (!globalGroup) throw new Error("Failed to load global group")
return globalGroup
}
/**
* Processes JSON files to extract link and topic data.
* @returns A Promise that resolves to a tuple containing a LinkManager and an array of TopicJson.
*/
async function processJsonFiles(): Promise<[LinkManager, TopicJson[]]> {
const directory = path.join(__dirname, "..", "private", "data", "edgedb", "topics")
const linkManager = new LinkManager()
const processedData: TopicJson[] = []
let files = await fs.readdir(directory)
files.sort((a, b) => a.localeCompare(b)) // sort files alphabetically
files = files.slice(0, 1) // get only 1 file for testing
for (const file of files) {
if (path.extname(file) === ".json") {
const filePath = path.join(directory, file)
try {
const data = JSON.parse(await fs.readFile(filePath, "utf-8")) as TopicJson
if (data.latestGlobalGuide) {
for (const section of data.latestGlobalGuide.sections) {
for (const link of section.links) {
linkManager.addLink(link)
}
}
}
processedData.push(data)
} catch (error) {
console.error(`Error processing file ${file}:`, error)
}
}
}
return [linkManager, processedData]
}
/**
* Creates a simple progress bar string.
* @param progress - Current progress (0-100).
* @param total - Total width of the progress bar.
* @returns A string representing the progress bar.
*/
function createProgressBar(progress: number, total: number = 30): string {
const filledWidth = Math.round((progress / 100) * total)
const emptyWidth = total - filledWidth
return `[${"=".repeat(filledWidth)}${" ".repeat(emptyWidth)}]`
}
/**
* Updates the progress display in the terminal.
* @param message - The message to display.
* @param current - Current progress value.
* @param total - Total progress value.
*/
function updateProgress(message: string, current: number, total: number) {
const percentage = Math.round((current / total) * 100)
const progressBar = createProgressBar(percentage)
process.stdout.write(`\r${message} ${progressBar} ${percentage}% (${current}/${total})`)
}
async function insertLinksInBatch(links: LinkJson[], chunkSize: number = 100) {
const globalGroup = await loadGlobalGroup()
const allCreatedLinks: Link[] = []
const totalLinks = links.length
for (let i = 0; i < totalLinks; i += chunkSize) {
const chunk = links.slice(i, i + chunkSize)
const rows = chunk.map(link =>
Link.create(
{
title: link.title,
url: link.url
},
{ owner: globalGroup }
)
)
allCreatedLinks.push(...rows)
updateProgress("Processing links:", i + chunk.length, totalLinks)
// Add a small delay between chunks to avoid overwhelming the system
await new Promise(resolve => setTimeout(resolve, 1000))
}
console.log("\nFinished processing links")
return allCreatedLinks
}
async function saveProcessedData(linkLists: Link[], topics: TopicJson[], chunkSize: number = 10) {
const globalGroup = await loadGlobalGroup()
const totalTopics = topics.length
for (let i = 0; i < totalTopics; i += chunkSize) {
const topicChunk = topics.slice(i, i + chunkSize)
topicChunk.forEach(topic => {
const topicModel = Topic.create(
{
name: topic.name,
prettyName: topic.prettyName,
latestGlobalGuide: LatestGlobalGuide.create(
{
sections: ListOfSections.create([], { owner: globalGroup })
},
{ owner: globalGroup }
)
},
{ owner: globalGroup }
)
if (!topic.latestGlobalGuide) {
console.error("No sections found in", topic.name)
return
}
topic.latestGlobalGuide.sections.map(section => {
const sectionModel = Section.create(
{
title: section.title,
links: ListOfLinks.create([], { owner: globalGroup })
},
{ owner: globalGroup }
)
section.links.map(link => {
const linkModel = linkLists.find(l => l.url === link.url)
if (linkModel) {
sectionModel.links?.push(linkModel)
}
})
topicModel.latestGlobalGuide?.sections?.push(sectionModel)
})
globalGroup.root.topics?.push(topicModel)
})
updateProgress("Processing topics:", i + topicChunk.length, totalTopics)
// Add a small delay between chunks to avoid overwhelming the system
await new Promise(resolve => setTimeout(resolve, 1000))
}
console.log("\nFinished processing topics")
}
/**
* Seeds production data.
*/
async function prodSeed() {
console.log("Starting to seed data")
const [linkManager, processedData] = await processJsonFiles()
console.log(`Collected ${linkManager.getAllLinks().length} unique links.`)
console.log(`Found ${linkManager.getDuplicateCount()} duplicate links.`)
console.log("\nInserting links:")
const insertedLinks = await insertLinksInBatch(linkManager.getAllLinks(), 100)
console.log("\nSaving processed data:")
await saveProcessedData(insertedLinks, processedData, 10)
console.log("\nFinished seeding data")
}
interface ForceGraphJson {
name: string
prettyName: string
connections: string[]
}
/**
* Manages links, handling deduplication and tracking duplicates.
*/
class ConnectionManager {
private connections: Map<string, string> = new Map()
private duplicateCount: number = 0
/**
* Adds a connection to the manager, tracking duplicates.
* @param connection - The connection to add.
*/
addConnection(connection: string) {
if (this.connections.has(connection)) {
this.duplicateCount++
} else {
this.connections.set(connection, connection)
}
}
/**
* Gets all unique connections.
* @returns An array of unique connections.
*/
getAllConnections() {
return Array.from(this.connections.values())
}
/**
* Gets the count of duplicate connections.
* @returns The number of duplicate connections.
*/
getDuplicateCount() {
return this.duplicateCount
}
}
/**
* Inserts connections in batch.
* @param connections - An array of string objects to insert.
* @returns A Promise that resolves to an array of created Connection models.
*/
async function insertConnectionsInBatch(connections: string[]) {
const globalGroup = await loadGlobalGroup()
const rows = []
for (const connection of connections) {
const connectionModel = Connection.create(
{
name: connection
},
{ owner: globalGroup }
)
rows.push(connectionModel)
}
return rows
}
/**
* Saves force graph data to the global group.
* @param connectionLists - An array of Connection models.
* @param forceGraphs - An array of ForceGraphJson objects.
*/
async function saveForceGraph(connectionLists: Connection[], forceGraphs: ForceGraphJson[]) {
const globalGroup = await loadGlobalGroup()
forceGraphs.map(forceGraph => {
const forceGraphModel = ForceGraph.create(
{
name: forceGraph.name,
prettyName: forceGraph.prettyName,
connections: ListOfConnections.create([], { owner: globalGroup })
},
{ owner: globalGroup }
)
forceGraph.connections.map(connection => {
const connectionModel = connectionLists.find(c => c.name === connection)
if (connectionModel) {
forceGraphModel.connections?.push(connectionModel)
}
})
globalGroup.root.forceGraphs?.push(forceGraphModel)
})
}
async function forceGraphSeed() {
console.log("Starting to seed force graph data")
const directory = path.join(__dirname, "..", "private", "data", "edgedb")
const connectionManager = new ConnectionManager()
const processedData: ForceGraphJson[] = []
const files = await fs.readdir(directory)
const file = files.find(file => file === "force-graphs.json")
if (!file) {
console.error("No force-graphs.json file found")
return
}
const filePath = path.join(directory, file)
try {
const forceGraphs = JSON.parse(await fs.readFile(filePath, "utf-8")) as ForceGraphJson[]
for (const forceGraph of forceGraphs) {
if (forceGraph.connections.length) {
for (const connection of forceGraph.connections) {
connectionManager.addConnection(connection)
}
}
processedData.push(forceGraph)
}
} catch (error) {
console.error(`Error processing file ${file}:`, error)
}
console.log(`Collected ${connectionManager.getAllConnections().length} unique connections.`)
console.log(`Found ${connectionManager.getDuplicateCount()} duplicate connections.`)
const insertedConnections = await insertConnectionsInBatch(connectionManager.getAllConnections())
await saveForceGraph(insertedConnections, processedData)
// wait 3 seconds before finishing
await new Promise(resolve => setTimeout(resolve, 3000))
console.log("Finished seeding force graph data")
}
/**
* Performs a full production rebuild.
*/
async function fullProdRebuild() {
await prodSeed()
await forceGraphSeed()
}
/**
* Main seed function to handle different commands.
*/
async function seed() {
const args = Bun.argv
const command = args[2]
try {
switch (command) {
case undefined:
console.log("No command provided")
break
case "setup":
await setup()
break
case "prod":
await prodSeed()
break
case "fullProdRebuild":
await fullProdRebuild()
break
case "forceGraph":
await forceGraphSeed()
break
default:
console.log("Unknown command")
break
}
} catch (err) {
console.error("Error occurred:", err)
}
}
await seed()