mirror of
https://github.com/linsa-io/linsa.git
synced 2026-01-11 22:40:32 +01:00
cleanup
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -14,3 +14,5 @@ private
|
||||
private.*
|
||||
private-*
|
||||
past.*
|
||||
past-*
|
||||
scripts/run.ts
|
||||
|
||||
@@ -6,6 +6,6 @@ resolver = "2"
|
||||
authors = ["Alice Carroll <learn-anything@alice-carroll.pet>"]
|
||||
edition = "2021"
|
||||
license-file = "license"
|
||||
repository = "https://github.com/learn-anything/learn-anything.xyz"
|
||||
repository = "https://github.com/learn-anything/learn-anything"
|
||||
rust-version = "1.80"
|
||||
version = "0.1.0"
|
||||
|
||||
6
api/.gitignore
vendored
6
api/.gitignore
vendored
@@ -1,6 +0,0 @@
|
||||
.encore
|
||||
encore.gen.go
|
||||
encore.gen.cue
|
||||
/.encore
|
||||
node_modules
|
||||
/encore.gen
|
||||
@@ -1,72 +0,0 @@
|
||||
import { api, APIError } from "encore.dev/api"
|
||||
// import { startWorker } from "jazz-nodejs"
|
||||
// import { ID } from "jazz-tools"
|
||||
import { secret } from "encore.dev/config"
|
||||
import log from "encore.dev/log"
|
||||
|
||||
const jazzWorkerAccountId = secret("jazzWorkerAccountId")
|
||||
const jazzWorkerSecret = secret("jazzWorkerSecret")
|
||||
const jazzPublicGlobalGroup = secret("jazzPublicGlobalGroup")
|
||||
|
||||
export const testRoute = api(
|
||||
{ expose: true, method: "GET", path: "/test" },
|
||||
async ({}: {}): Promise<void> => {
|
||||
console.log(jazzPublicGlobalGroup(), "group")
|
||||
log.info("better logs from encore")
|
||||
}
|
||||
)
|
||||
|
||||
// return all content for GlobalTopic
|
||||
export const getTopic = api(
|
||||
{ expose: true, method: "GET", path: "/topic/:topic" },
|
||||
async ({
|
||||
topic
|
||||
}: {
|
||||
topic: string
|
||||
// TODO: can return type be inferred like Elysia?
|
||||
}): Promise<{
|
||||
links: {
|
||||
label: string
|
||||
url: string
|
||||
}[]
|
||||
}> => {
|
||||
// const { worker } = await startWorker({
|
||||
// accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
|
||||
// accountSecret: JAZZ_WORKER_SECRET
|
||||
// })
|
||||
|
||||
// TODO: how to get the import from outside this package?
|
||||
// const globalGroupId = process.env.JAZZ_PUBLIC_GLOBAL_GROUP as ID<any>
|
||||
// const globalGroup = await PublicGlobalGroup.load(globalGroupId, worker, {
|
||||
// root: {
|
||||
// topics: [
|
||||
// {
|
||||
// latestGlobalGuide: {
|
||||
// sections: [
|
||||
// {
|
||||
// links: [{}]
|
||||
// }
|
||||
// ]
|
||||
// }
|
||||
// }
|
||||
// ],
|
||||
// forceGraphs: [
|
||||
// {
|
||||
// connections: [{}]
|
||||
// }
|
||||
// ]
|
||||
// }
|
||||
// })
|
||||
// if (!globalGroup) throw APIError.notFound("GlobalGroup not found")
|
||||
|
||||
// const globalGroupId = process.env.JAZZ_PUBLIC_GLOBAL_GROUP as ID<any>
|
||||
// console.log(globalGroupId)
|
||||
// console.log(worker)
|
||||
// console.log("runs..")
|
||||
|
||||
const topicContent = {
|
||||
links: []
|
||||
}
|
||||
return topicContent
|
||||
}
|
||||
)
|
||||
@@ -1,40 +0,0 @@
|
||||
// TODO: not sure if `links` should be separate service
|
||||
// it is responsible for adding and getting links into LA from API
|
||||
|
||||
import { api, APIError } from "encore.dev/api"
|
||||
// import { startWorker } from "jazz-nodejs"
|
||||
import { secret } from "encore.dev/config"
|
||||
|
||||
const jazzWorkerSecret = secret("jazzWorkerSecret")
|
||||
|
||||
export const addPersonalLink = api(
|
||||
{ expose: true, method: "POST", path: "/save-link" },
|
||||
async ({ url }: { url: string }): Promise<void> => {
|
||||
// const { worker } = await startWorker({
|
||||
// accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
|
||||
// accountSecret: JAZZ_WORKER_SECRET
|
||||
// })
|
||||
}
|
||||
)
|
||||
|
||||
export const getLinkDetails = api(
|
||||
{ expose: true, method: "GET", path: "/global-link-details/:url" },
|
||||
async ({
|
||||
url
|
||||
}: {
|
||||
url: string
|
||||
}): Promise<{
|
||||
title: string
|
||||
summary?: string
|
||||
}> => {
|
||||
// const { worker } = await startWorker({
|
||||
// accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
|
||||
// accountSecret: JAZZ_WORKER_SECRET
|
||||
// })
|
||||
|
||||
return {
|
||||
title: "Jazz",
|
||||
summary: "Jazz is local first framework for building web apps"
|
||||
}
|
||||
}
|
||||
)
|
||||
BIN
api/bun.lockb
BIN
api/bun.lockb
Binary file not shown.
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"id": "encore-test-76k2",
|
||||
"lang": "typescript"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "api",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "encore run"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.5.4",
|
||||
"typescript": "^5.6.2",
|
||||
"vitest": "^2.0.5"
|
||||
},
|
||||
"dependencies": {
|
||||
"encore.dev": "^1.41.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@rollup/rollup-linux-x64-gnu": "^4.21.2"
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
Using [Encore](https://encore.dev).
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"compilerOptions": {
|
||||
/* Basic Options */
|
||||
"lib": ["ES2022"],
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"types": ["node"],
|
||||
"paths": {
|
||||
"~encore/*": ["./encore.gen/*"]
|
||||
},
|
||||
|
||||
/* Workspace Settings */
|
||||
"composite": true,
|
||||
|
||||
/* Strict Type-Checking Options */
|
||||
"strict": true,
|
||||
|
||||
/* Module Resolution Options */
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"isolatedModules": true,
|
||||
"sourceMap": true,
|
||||
|
||||
"declaration": true,
|
||||
|
||||
/* Advanced Options */
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
# CLI
|
||||
|
||||
> CLI for interfacing with LA
|
||||
|
||||
Will be modelled after [Encore's Go CLI](https://github.com/encoredev/encore/tree/main/cli/cmd/encore).
|
||||
@@ -4,8 +4,6 @@
|
||||
|
||||
### Files
|
||||
|
||||
- [lib](lib) - shared functions
|
||||
- [scripts](scripts) - utility scripts
|
||||
- [web](web) - website hosted on [learn-anything.xyz](https://learn-anything.xyz) (using [React](https://react.dev/), [TanStack Start](https://tanstack.com/start/latest), [Jazz](https://jazz.tools/))
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
// @ts-nocheck
|
||||
async function devSeed() {
|
||||
const { worker } = await startWorker({
|
||||
accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
|
||||
accountSecret: JAZZ_WORKER_SECRET
|
||||
})
|
||||
const user = (await (
|
||||
await LaAccount.createAs(worker, {
|
||||
creationProps: { name: "nikiv" }
|
||||
})
|
||||
).ensureLoaded({ root: { personalLinks: [], pages: [], todos: [] } }))!
|
||||
const globalLinksGroup = Group.create({ owner: worker })
|
||||
globalLinksGroup.addMember("everyone", "reader")
|
||||
const globalLink1 = GlobalLink.create({ url: "https://google.com" }, { owner: globalLinksGroup })
|
||||
const globalLink2 = GlobalLink.create({ url: "https://jazz.tools" }, { owner: globalLinksGroup })
|
||||
// TODO: make note: optional
|
||||
const personalLink1 = PersonalLink.create(
|
||||
{ globalLink: globalLink1, type: "personalLink", note: "" },
|
||||
{ owner: user }
|
||||
)
|
||||
const personalLink2 = PersonalLink.create(
|
||||
{ globalLink: globalLink2, type: "personalLink", note: "Great framework" },
|
||||
{ owner: user }
|
||||
)
|
||||
user.root.personalLinks.push(personalLink1)
|
||||
user.root.personalLinks.push(personalLink2)
|
||||
const pageOneTitle = "Physics"
|
||||
const pageTwoTitle = "Karabiner"
|
||||
const page1 = PersonalPage.create(
|
||||
{ title: pageOneTitle, slug: generateUniqueSlug(pageOneTitle), content: "Physics is great" },
|
||||
{ owner: user }
|
||||
)
|
||||
const page2 = PersonalPage.create(
|
||||
{ title: pageTwoTitle, slug: generateUniqueSlug(pageTwoTitle), content: "Karabiner is great" },
|
||||
{ owner: user }
|
||||
)
|
||||
user.root.personalPages?.push(page1)
|
||||
user.root.personalPages?.push(page2)
|
||||
const page1 = Page.create({ title: "Physics", content: "Physics is great" }, { owner: user })
|
||||
const page2 = Page.create({ title: "Karabiner", content: "Karabiner is great" }, { owner: user })
|
||||
user.root.pages.push(page1)
|
||||
user.root.pages.push(page2)
|
||||
const credentials = {
|
||||
accountID: user.id,
|
||||
accountSecret: (user._raw as RawControlledAccount).agentSecret
|
||||
}
|
||||
await Bun.write(
|
||||
"./web/.env",
|
||||
`VITE_SEED_ACCOUNTS='${JSON.stringify({
|
||||
nikiv: credentials
|
||||
})}'`
|
||||
)
|
||||
await Bun.write(
|
||||
"./.env",
|
||||
`VITE_SEED_ACCOUNTS='${JSON.stringify({
|
||||
nikiv: credentials
|
||||
})}'`
|
||||
)
|
||||
}
|
||||
const globalLink = GlobalLink.create(
|
||||
{
|
||||
url: "https://google.com",
|
||||
urlTitle: "Google",
|
||||
protocol: "https"
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
const user = (await (
|
||||
await LaAccount.createAs(worker, {
|
||||
creationProps: { name: "nikiv" }
|
||||
})
|
||||
).ensureLoaded({ root: { personalLinks: [], pages: [], todos: [] } }))!
|
||||
console.log(process.env.JAZZ_GLOBAL_GROUP!, "group")
|
||||
console.log(worker)
|
||||
// TODO: type err
|
||||
console.log(globalGroup, "group")
|
||||
return
|
||||
const currentFilePath = import.meta.path
|
||||
const connectionsFilePath = `${currentFilePath.replace("seed.ts", "/seed/connections.json")}`
|
||||
const file = Bun.file(connectionsFilePath)
|
||||
const fileContent = await file.text()
|
||||
const topicsWithConnections = JSON.parse(fileContent)
|
||||
// let topicsWithConnections = JSON.stringify(obj, null, 2)
|
||||
console.log(topicsWithConnections)
|
||||
// TODO: type err
|
||||
topicsWithConnections.map(topic => {
|
||||
const globalTopic = GlobalTopic.create({ name: topic.name, description: topic.description }, { owner: globalGroup })
|
||||
})
|
||||
@@ -1,89 +0,0 @@
|
||||
import { getEnvOrThrow } from "@/lib/utils"
|
||||
import { PublicGlobalGroup } from "@/web/lib/schema/master/public-group"
|
||||
import { startWorker } from "jazz-nodejs"
|
||||
import { ID } from "jazz-tools"
|
||||
|
||||
const JAZZ_WORKER_SECRET = getEnvOrThrow("JAZZ_WORKER_SECRET")
|
||||
|
||||
async function run() {
|
||||
try {
|
||||
await readJazz()
|
||||
} catch (err) {
|
||||
console.log(err, "err")
|
||||
}
|
||||
}
|
||||
|
||||
async function readJazz() {
|
||||
const { worker } = await startWorker({
|
||||
accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
|
||||
accountSecret: JAZZ_WORKER_SECRET
|
||||
})
|
||||
|
||||
const globalGroupId = process.env.JAZZ_PUBLIC_GLOBAL_GROUP as ID<PublicGlobalGroup>
|
||||
const globalGroup = await PublicGlobalGroup.load(globalGroupId, worker, {
|
||||
root: {
|
||||
topics: [
|
||||
{
|
||||
latestGlobalGuide: {
|
||||
sections: [
|
||||
{
|
||||
links: [{}]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
forceGraphs: [
|
||||
{
|
||||
connections: [{}]
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
if (!globalGroup) return // TODO: err
|
||||
|
||||
// wait 10 seconds
|
||||
await new Promise(resolve => setTimeout(resolve, 10000))
|
||||
|
||||
/*
|
||||
* Log forceGraphs
|
||||
*/
|
||||
const asJsonForceGraphs = globalGroup.root.forceGraphs.map(node => {
|
||||
console.log({ node }, "node")
|
||||
return {
|
||||
name: node.name,
|
||||
prettyName: node.prettyName,
|
||||
connections: node.connections?.map(connection => {
|
||||
return {
|
||||
name: connection?.name
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const asJson = globalGroup.root.topics?.map(node => {
|
||||
return {
|
||||
name: node.name,
|
||||
prettyName: node.prettyName,
|
||||
latestGlobalGuide: {
|
||||
sections: node.latestGlobalGuide.sections.map(section => {
|
||||
return {
|
||||
title: section?.title,
|
||||
links: section?.links?.map(link => {
|
||||
return {
|
||||
title: link?.title,
|
||||
url: link?.url
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
console.log({ asJsonForceGraphs }, "asJsonForceGraphs")
|
||||
console.log({ asJson }, "asJson")
|
||||
}
|
||||
|
||||
await run()
|
||||
493
scripts/seed.ts
493
scripts/seed.ts
@@ -1,493 +0,0 @@
|
||||
import { getEnvOrThrow } from "@/lib/utils"
|
||||
import { Connection, ForceGraph, ListOfConnections, ListOfForceGraphs } from "@/web/lib/schema/master/force-graph"
|
||||
import { PublicGlobalGroup, PublicGlobalGroupRoot } from "@/web/lib/schema/master/public-group"
|
||||
import {
|
||||
LatestGlobalGuide,
|
||||
Link,
|
||||
ListOfLinks,
|
||||
ListOfSections,
|
||||
ListOfTopics,
|
||||
Section,
|
||||
Topic
|
||||
} from "@/web/lib/schema/master/topic"
|
||||
import fs from "fs/promises"
|
||||
import { startWorker } from "jazz-nodejs"
|
||||
import { ID } from "jazz-tools"
|
||||
import { appendFile } from "node:fs/promises"
|
||||
import path from "path"
|
||||
|
||||
// Define interfaces for JSON data structures
|
||||
interface LinkJson {
|
||||
id?: ID<Link>
|
||||
title: string
|
||||
url: string
|
||||
}
|
||||
|
||||
interface SectionJson {
|
||||
title: string
|
||||
links: LinkJson[]
|
||||
}
|
||||
|
||||
interface TopicJson {
|
||||
name: string
|
||||
prettyName: string
|
||||
latestGlobalGuide: {
|
||||
sections: SectionJson[]
|
||||
} | null
|
||||
}
|
||||
|
||||
// Get the Jazz worker secret from environment variables
|
||||
const JAZZ_WORKER_SECRET = getEnvOrThrow("JAZZ_WORKER_SECRET")
|
||||
|
||||
/**
|
||||
* Manages links, handling deduplication and tracking duplicates.
|
||||
*/
|
||||
class LinkManager {
|
||||
private links: Map<string, LinkJson> = new Map()
|
||||
private duplicateCount: number = 0
|
||||
|
||||
/**
|
||||
* Adds a link to the manager, tracking duplicates.
|
||||
* @param link - The link to add.
|
||||
*/
|
||||
addLink(link: LinkJson) {
|
||||
const key = link.url
|
||||
if (this.links.has(key)) {
|
||||
this.duplicateCount++
|
||||
} else {
|
||||
this.links.set(key, link)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all unique links.
|
||||
* @returns An array of unique links.
|
||||
*/
|
||||
getAllLinks() {
|
||||
return Array.from(this.links.values())
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the count of duplicate links.
|
||||
* @returns The number of duplicate links.
|
||||
*/
|
||||
getDuplicateCount() {
|
||||
return this.duplicateCount
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts a Jazz worker.
|
||||
* @returns A Promise that resolves to the started worker.
|
||||
*/
|
||||
async function startJazzWorker() {
|
||||
const { worker } = await startWorker({
|
||||
accountID: "co_zhvp7ryXJzDvQagX61F6RCZFJB9",
|
||||
accountSecret: JAZZ_WORKER_SECRET
|
||||
})
|
||||
return worker
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up the global and admin groups.
|
||||
*/
|
||||
async function setup() {
|
||||
console.log("Starting setup")
|
||||
|
||||
const worker = await startJazzWorker()
|
||||
|
||||
/*
|
||||
* Create global group
|
||||
*/
|
||||
const publicGlobalGroup = PublicGlobalGroup.create({ owner: worker })
|
||||
publicGlobalGroup.root = PublicGlobalGroupRoot.create(
|
||||
{
|
||||
topics: ListOfTopics.create([], { owner: publicGlobalGroup }),
|
||||
forceGraphs: ListOfForceGraphs.create([], { owner: publicGlobalGroup })
|
||||
},
|
||||
{ owner: publicGlobalGroup }
|
||||
)
|
||||
publicGlobalGroup.addMember("everyone", "reader")
|
||||
await appendFile("./.env", `\nJAZZ_PUBLIC_GLOBAL_GROUP=${JSON.stringify(publicGlobalGroup.id)}`)
|
||||
|
||||
/*
|
||||
* Create admin group
|
||||
*/
|
||||
// const user = (await await LaAccount.createAs(worker, {
|
||||
// creationProps: { name: "nikiv" }
|
||||
// }))!
|
||||
// const adminGlobalGroup = Group.create({ owner: worker })
|
||||
// adminGlobalGroup.addMember(user, "admin")
|
||||
// await appendFile("./.env", `\nJAZZ_ADMIN_GLOBAL_GROUP=${JSON.stringify(adminGlobalGroup.id)}`)
|
||||
|
||||
console.log("Setup completed successfully", publicGlobalGroup.id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the global group.
|
||||
* @returns A Promise that resolves to the loaded global group.
|
||||
* @throws Error if the global group fails to load.
|
||||
*/
|
||||
async function loadGlobalGroup() {
|
||||
const worker = await startJazzWorker()
|
||||
|
||||
const globalGroupId = getEnvOrThrow("JAZZ_PUBLIC_GLOBAL_GROUP") as ID<PublicGlobalGroup>
|
||||
|
||||
const globalGroup = await PublicGlobalGroup.load(globalGroupId, worker, {
|
||||
root: {
|
||||
topics: [{ latestGlobalGuide: { sections: [] } }],
|
||||
forceGraphs: [{ connections: [] }]
|
||||
}
|
||||
})
|
||||
|
||||
if (!globalGroup) throw new Error("Failed to load global group")
|
||||
|
||||
return globalGroup
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes JSON files to extract link and topic data.
|
||||
* @returns A Promise that resolves to a tuple containing a LinkManager and an array of TopicJson.
|
||||
*/
|
||||
async function processJsonFiles(): Promise<[LinkManager, TopicJson[]]> {
|
||||
const directory = path.join(__dirname, "..", "private", "data", "edgedb", "topics")
|
||||
|
||||
const linkManager = new LinkManager()
|
||||
const processedData: TopicJson[] = []
|
||||
|
||||
let files = await fs.readdir(directory)
|
||||
files.sort((a, b) => a.localeCompare(b)) // sort files alphabetically
|
||||
|
||||
files = files.slice(0, 1) // get only 1 file for testing
|
||||
|
||||
for (const file of files) {
|
||||
if (path.extname(file) === ".json") {
|
||||
const filePath = path.join(directory, file)
|
||||
try {
|
||||
const data = JSON.parse(await fs.readFile(filePath, "utf-8")) as TopicJson
|
||||
if (data.latestGlobalGuide) {
|
||||
for (const section of data.latestGlobalGuide.sections) {
|
||||
for (const link of section.links) {
|
||||
linkManager.addLink(link)
|
||||
}
|
||||
}
|
||||
}
|
||||
processedData.push(data)
|
||||
} catch (error) {
|
||||
console.error(`Error processing file ${file}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [linkManager, processedData]
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a simple progress bar string.
|
||||
* @param progress - Current progress (0-100).
|
||||
* @param total - Total width of the progress bar.
|
||||
* @returns A string representing the progress bar.
|
||||
*/
|
||||
function createProgressBar(progress: number, total: number = 30): string {
|
||||
const filledWidth = Math.round((progress / 100) * total)
|
||||
const emptyWidth = total - filledWidth
|
||||
return `[${"=".repeat(filledWidth)}${" ".repeat(emptyWidth)}]`
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the progress display in the terminal.
|
||||
* @param message - The message to display.
|
||||
* @param current - Current progress value.
|
||||
* @param total - Total progress value.
|
||||
*/
|
||||
function updateProgress(message: string, current: number, total: number) {
|
||||
const percentage = Math.round((current / total) * 100)
|
||||
const progressBar = createProgressBar(percentage)
|
||||
process.stdout.write(`\r${message} ${progressBar} ${percentage}% (${current}/${total})`)
|
||||
}
|
||||
|
||||
async function insertLinksInBatch(links: LinkJson[], chunkSize: number = 100) {
|
||||
const globalGroup = await loadGlobalGroup()
|
||||
const allCreatedLinks: Link[] = []
|
||||
const totalLinks = links.length
|
||||
|
||||
for (let i = 0; i < totalLinks; i += chunkSize) {
|
||||
const chunk = links.slice(i, i + chunkSize)
|
||||
const rows = chunk.map(link =>
|
||||
Link.create(
|
||||
{
|
||||
title: link.title,
|
||||
url: link.url
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
)
|
||||
allCreatedLinks.push(...rows)
|
||||
|
||||
updateProgress("Processing links:", i + chunk.length, totalLinks)
|
||||
|
||||
// Add a small delay between chunks to avoid overwhelming the system
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
|
||||
console.log("\nFinished processing links")
|
||||
return allCreatedLinks
|
||||
}
|
||||
|
||||
async function saveProcessedData(linkLists: Link[], topics: TopicJson[], chunkSize: number = 10) {
|
||||
const globalGroup = await loadGlobalGroup()
|
||||
const totalTopics = topics.length
|
||||
|
||||
for (let i = 0; i < totalTopics; i += chunkSize) {
|
||||
const topicChunk = topics.slice(i, i + chunkSize)
|
||||
|
||||
topicChunk.forEach(topic => {
|
||||
const topicModel = Topic.create(
|
||||
{
|
||||
name: topic.name,
|
||||
prettyName: topic.prettyName,
|
||||
latestGlobalGuide: LatestGlobalGuide.create(
|
||||
{
|
||||
sections: ListOfSections.create([], { owner: globalGroup })
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
|
||||
if (!topic.latestGlobalGuide) {
|
||||
console.error("No sections found in", topic.name)
|
||||
return
|
||||
}
|
||||
|
||||
topic.latestGlobalGuide.sections.map(section => {
|
||||
const sectionModel = Section.create(
|
||||
{
|
||||
title: section.title,
|
||||
links: ListOfLinks.create([], { owner: globalGroup })
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
|
||||
section.links.map(link => {
|
||||
const linkModel = linkLists.find(l => l.url === link.url)
|
||||
if (linkModel) {
|
||||
sectionModel.links?.push(linkModel)
|
||||
}
|
||||
})
|
||||
|
||||
topicModel.latestGlobalGuide?.sections?.push(sectionModel)
|
||||
})
|
||||
|
||||
globalGroup.root.topics?.push(topicModel)
|
||||
})
|
||||
|
||||
updateProgress("Processing topics:", i + topicChunk.length, totalTopics)
|
||||
|
||||
// Add a small delay between chunks to avoid overwhelming the system
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
|
||||
console.log("\nFinished processing topics")
|
||||
}
|
||||
|
||||
/**
|
||||
* Seeds production data.
|
||||
*/
|
||||
async function prodSeed() {
|
||||
console.log("Starting to seed data")
|
||||
|
||||
const [linkManager, processedData] = await processJsonFiles()
|
||||
|
||||
console.log(`Collected ${linkManager.getAllLinks().length} unique links.`)
|
||||
console.log(`Found ${linkManager.getDuplicateCount()} duplicate links.`)
|
||||
|
||||
console.log("\nInserting links:")
|
||||
const insertedLinks = await insertLinksInBatch(linkManager.getAllLinks(), 100)
|
||||
|
||||
console.log("\nSaving processed data:")
|
||||
await saveProcessedData(insertedLinks, processedData, 10)
|
||||
|
||||
console.log("\nFinished seeding data")
|
||||
}
|
||||
|
||||
interface ForceGraphJson {
|
||||
name: string
|
||||
prettyName: string
|
||||
connections: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages links, handling deduplication and tracking duplicates.
|
||||
*/
|
||||
class ConnectionManager {
|
||||
private connections: Map<string, string> = new Map()
|
||||
private duplicateCount: number = 0
|
||||
|
||||
/**
|
||||
* Adds a connection to the manager, tracking duplicates.
|
||||
* @param connection - The connection to add.
|
||||
*/
|
||||
addConnection(connection: string) {
|
||||
if (this.connections.has(connection)) {
|
||||
this.duplicateCount++
|
||||
} else {
|
||||
this.connections.set(connection, connection)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all unique connections.
|
||||
* @returns An array of unique connections.
|
||||
*/
|
||||
getAllConnections() {
|
||||
return Array.from(this.connections.values())
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the count of duplicate connections.
|
||||
* @returns The number of duplicate connections.
|
||||
*/
|
||||
getDuplicateCount() {
|
||||
return this.duplicateCount
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts connections in batch.
|
||||
* @param connections - An array of string objects to insert.
|
||||
* @returns A Promise that resolves to an array of created Connection models.
|
||||
*/
|
||||
async function insertConnectionsInBatch(connections: string[]) {
|
||||
const globalGroup = await loadGlobalGroup()
|
||||
const rows = []
|
||||
|
||||
for (const connection of connections) {
|
||||
const connectionModel = Connection.create(
|
||||
{
|
||||
name: connection
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
rows.push(connectionModel)
|
||||
}
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves force graph data to the global group.
|
||||
* @param connectionLists - An array of Connection models.
|
||||
* @param forceGraphs - An array of ForceGraphJson objects.
|
||||
*/
|
||||
async function saveForceGraph(connectionLists: Connection[], forceGraphs: ForceGraphJson[]) {
|
||||
const globalGroup = await loadGlobalGroup()
|
||||
|
||||
forceGraphs.map(forceGraph => {
|
||||
const forceGraphModel = ForceGraph.create(
|
||||
{
|
||||
name: forceGraph.name,
|
||||
prettyName: forceGraph.prettyName,
|
||||
connections: ListOfConnections.create([], { owner: globalGroup })
|
||||
},
|
||||
{ owner: globalGroup }
|
||||
)
|
||||
|
||||
forceGraph.connections.map(connection => {
|
||||
const connectionModel = connectionLists.find(c => c.name === connection)
|
||||
if (connectionModel) {
|
||||
forceGraphModel.connections?.push(connectionModel)
|
||||
}
|
||||
})
|
||||
|
||||
globalGroup.root.forceGraphs?.push(forceGraphModel)
|
||||
})
|
||||
}
|
||||
|
||||
async function forceGraphSeed() {
|
||||
console.log("Starting to seed force graph data")
|
||||
|
||||
const directory = path.join(__dirname, "..", "private", "data", "edgedb")
|
||||
|
||||
const connectionManager = new ConnectionManager()
|
||||
const processedData: ForceGraphJson[] = []
|
||||
|
||||
const files = await fs.readdir(directory)
|
||||
const file = files.find(file => file === "force-graphs.json")
|
||||
|
||||
if (!file) {
|
||||
console.error("No force-graphs.json file found")
|
||||
return
|
||||
}
|
||||
|
||||
const filePath = path.join(directory, file)
|
||||
|
||||
try {
|
||||
const forceGraphs = JSON.parse(await fs.readFile(filePath, "utf-8")) as ForceGraphJson[]
|
||||
|
||||
for (const forceGraph of forceGraphs) {
|
||||
if (forceGraph.connections.length) {
|
||||
for (const connection of forceGraph.connections) {
|
||||
connectionManager.addConnection(connection)
|
||||
}
|
||||
}
|
||||
|
||||
processedData.push(forceGraph)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error processing file ${file}:`, error)
|
||||
}
|
||||
|
||||
console.log(`Collected ${connectionManager.getAllConnections().length} unique connections.`)
|
||||
console.log(`Found ${connectionManager.getDuplicateCount()} duplicate connections.`)
|
||||
|
||||
const insertedConnections = await insertConnectionsInBatch(connectionManager.getAllConnections())
|
||||
await saveForceGraph(insertedConnections, processedData)
|
||||
|
||||
// wait 3 seconds before finishing
|
||||
await new Promise(resolve => setTimeout(resolve, 3000))
|
||||
console.log("Finished seeding force graph data")
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs a full production rebuild.
|
||||
*/
|
||||
async function fullProdRebuild() {
|
||||
await prodSeed()
|
||||
await forceGraphSeed()
|
||||
}
|
||||
|
||||
/**
|
||||
* Main seed function to handle different commands.
|
||||
*/
|
||||
async function seed() {
|
||||
const args = Bun.argv
|
||||
const command = args[2]
|
||||
try {
|
||||
switch (command) {
|
||||
case undefined:
|
||||
console.log("No command provided")
|
||||
break
|
||||
case "setup":
|
||||
await setup()
|
||||
break
|
||||
case "prod":
|
||||
await prodSeed()
|
||||
break
|
||||
case "fullProdRebuild":
|
||||
await fullProdRebuild()
|
||||
break
|
||||
case "forceGraph":
|
||||
await forceGraphSeed()
|
||||
break
|
||||
default:
|
||||
console.log("Unknown command")
|
||||
break
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error occurred:", err)
|
||||
}
|
||||
}
|
||||
|
||||
await seed()
|
||||
@@ -1,12 +0,0 @@
|
||||
VITE_APP_NAME="Learn Anything"
|
||||
VITE_APP_URL=http://localhost:3000
|
||||
|
||||
VITE_CLERK_PUBLISHABLE_KEY=
|
||||
CLERK_SECRET_KEY=
|
||||
VITE_CLERK_SIGN_IN_URL=/sign-in
|
||||
VITE_CLERK_SIGN_UP_URL=/sign-up
|
||||
|
||||
VITE_JAZZ_PEER_URL="wss://mesh.jazz.tools/?key=example@gmail.com"
|
||||
VITE_JAZZ_GLOBAL_GROUP_ID=""
|
||||
|
||||
RONIN_TOKEN=
|
||||
@@ -1,94 +0,0 @@
|
||||
## Below is TanStack Start Clerk
|
||||
|
||||
Gotten from [here](https://github.com/TanStack/router/tree/main/examples/react/start-clerk-basic).
|
||||
|
||||
To make it work:
|
||||
|
||||
```
|
||||
bun i
|
||||
bun dev
|
||||
```
|
||||
|
||||
And have this in `.env`:
|
||||
|
||||
<!-- TODO: create some keys so others can also run it (not LA specific..) -->
|
||||
|
||||
You can get those keys from [Clerk](https://clerk.com/).
|
||||
|
||||
```
|
||||
CLERK_PUBLISHABLE_KEY=
|
||||
CLERK_SECRET_KEY=
|
||||
```
|
||||
|
||||
# Welcome to TanStack.com!
|
||||
|
||||
This site is built with TanStack Router!
|
||||
|
||||
- [TanStack Router Docs](https://tanstack.com/router)
|
||||
|
||||
It's deployed automagically with Vercel!
|
||||
|
||||
- [Vercel](https://vercel.com/)
|
||||
|
||||
## Development
|
||||
|
||||
From your terminal:
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
This starts your app in development mode, rebuilding assets on file changes.
|
||||
|
||||
## Editing and previewing the docs of TanStack projects locally
|
||||
|
||||
The documentations for all TanStack projects except for `React Charts` are hosted on [https://tanstack.com](https://tanstack.com), powered by this TanStack Router app.
|
||||
In production, the markdown doc pages are fetched from the GitHub repos of the projects, but in development they are read from the local file system.
|
||||
|
||||
Follow these steps if you want to edit the doc pages of a project (in these steps we'll assume it's [`TanStack/form`](https://github.com/tanstack/form)) and preview them locally :
|
||||
|
||||
1. Create a new directory called `tanstack`.
|
||||
|
||||
```sh
|
||||
mkdir tanstack
|
||||
```
|
||||
|
||||
2. Enter the directory and clone this repo and the repo of the project there.
|
||||
|
||||
```sh
|
||||
cd tanstack
|
||||
git clone git@github.com:TanStack/tanstack.com.git
|
||||
git clone git@github.com:TanStack/form.git
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Your `tanstack` directory should look like this:
|
||||
>
|
||||
> ```
|
||||
> tanstack/
|
||||
> |
|
||||
> +-- form/
|
||||
> |
|
||||
> +-- tanstack.com/
|
||||
> ```
|
||||
|
||||
> [!WARNING]
|
||||
> Make sure the name of the directory in your local file system matches the name of the project's repo. For example, `tanstack/form` must be cloned into `form` (this is the default) instead of `some-other-name`, because that way, the doc pages won't be found.
|
||||
|
||||
3. Enter the `tanstack/tanstack.com` directory, install the dependencies and run the app in dev mode:
|
||||
|
||||
```sh
|
||||
cd tanstack.com
|
||||
pnpm i
|
||||
# The app will run on https://localhost:3000 by default
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
4. Now you can visit http://localhost:3000/form/latest/docs/overview in the browser and see the changes you make in `tanstack/form/docs`.
|
||||
|
||||
> [!NOTE]
|
||||
> The updated pages need to be manually reloaded in the browser.
|
||||
|
||||
> [!WARNING]
|
||||
> You will need to update the `docs/config.json` file (in the project's repo) if you add a new doc page!
|
||||
Reference in New Issue
Block a user