Introduce pkl-doc model version 2 (#1169)

Currently, in order to update a pkl-doc documentation site,
almost the entire existing site is read in order to update metadata
like known versions, known subtypes, and more.

For example, adding a new version of a package requires that the
existing runtime data of all existing versions be updated.
Eventually, this causes the required storage size to balloon
exponentially to the number of versions.

This addresses these limitations by:

1. Updating the runtime data structure to move "known versions" metadata
   to the package level (the same JSON file is used for all versions).
2. Eliminating known subtype and known usage information at a
   cross-package level.
3. Generating the search index by consuming the previously generated
   search index.
4. Generating the main page by consuming the search index.

Because this changes how runtime data is stored, an existing docsite
needs to be migrated.

This also introduces a new migration command, `pkl-doc --migrate`,
which transforms an older version of the website into a newer version.
This commit is contained in:
Daniel Chao
2025-09-29 16:10:44 -07:00
committed by GitHub
parent 63f89fb679
commit 5d90cf8f4e
1599 changed files with 129992 additions and 581 deletions

View File

@@ -0,0 +1,28 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.doc
import java.io.OutputStream
abstract class AbstractGenerator(protected val consoleOut: OutputStream) {
protected fun writeOutputLine(message: String) {
consoleOut.writeLine(message)
}
protected fun writeOutput(message: String) {
consoleOut.write(message)
}
}

View File

@@ -15,6 +15,7 @@
*/
package org.pkl.doc
import java.io.OutputStream
import kotlinx.html.*
import org.pkl.core.PClass
@@ -25,7 +26,16 @@ internal class ClassPageGenerator(
clazz: PClass,
pageScope: ClassScope,
isTestMode: Boolean,
) : ModuleOrClassPageGenerator<ClassScope>(docsiteInfo, docModule, clazz, pageScope, isTestMode) {
consoleOut: OutputStream,
) :
ModuleOrClassPageGenerator<ClassScope>(
docsiteInfo,
docModule,
clazz,
pageScope,
isTestMode,
consoleOut,
) {
override val html: HTML.() -> Unit = {
renderHtmlHead()
@@ -56,12 +66,12 @@ internal class ClassPageGenerator(
clazz.annotations,
isDeclaration = true,
mapOf(
MemberInfoKey("Known subtypes", runtimeDataClasses) to
MemberInfoKey("Known subtypes in package", runtimeDataClasses) to
{
id = HtmlConstants.KNOWN_SUBTYPES
classes = runtimeDataClasses
},
MemberInfoKey("Known usages", runtimeDataClasses) to
MemberInfoKey("Known usages in package", runtimeDataClasses) to
{
id = HtmlConstants.KNOWN_USAGES
classes = runtimeDataClasses

View File

@@ -15,11 +15,13 @@
*/
package org.pkl.doc
import java.io.OutputStream
import java.net.URI
import java.net.URISyntaxException
import java.nio.file.Path
import kotlin.Pair
import org.pkl.commons.cli.CliBaseOptions.Companion.getProjectFile
import org.pkl.commons.cli.CliBugException
import org.pkl.commons.cli.CliCommand
import org.pkl.commons.cli.CliException
import org.pkl.commons.toPath
@@ -33,7 +35,11 @@ import org.pkl.core.packages.*
*
* For the low-level Pkldoc API, see [DocGenerator].
*/
class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(options.base) {
class CliDocGenerator(
private val options: CliDocGeneratorOptions,
private val consoleOut: OutputStream = System.out,
) : CliCommand(options.base) {
constructor(options: CliDocGeneratorOptions) : this(options, System.out)
private val packageResolver =
PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir)
@@ -60,6 +66,17 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
),
)
private val versions = mutableMapOf<String, Version>()
private val versionComparator =
Comparator<String> { v1, v2 ->
versions
.getOrPut(v1) { Version.parse(v1) }
.compareTo(versions.getOrPut(v2) { Version.parse(v2) })
}
private val docMigrator = DocMigrator(options.outputDir, System.out, versionComparator)
private fun DependencyMetadata.getPackageDependencies(): List<DocPackageInfo.PackageDependency> {
return buildList {
for ((_, dependency) in dependencies) {
@@ -87,14 +104,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
}
private fun PackageUri.toDocPackageInfo(): DocPackageInfo {
val metadataAndChecksum =
val (metadata, checksum) =
try {
packageResolver.getDependencyMetadataAndComputeChecksum(this)
} catch (e: PackageLoadError) {
throw CliException("Failed to package metadata for $this: ${e.message}")
}
val metadata = metadataAndChecksum.first
val checksum = metadataAndChecksum.second
return DocPackageInfo(
name = "${uri.authority}${uri.path.substringBeforeLast('@')}",
moduleNamePrefix = "${metadata.name}.",
@@ -130,6 +145,15 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
}
override fun doRun() {
if (options.migrate) {
docMigrator.run()
return
}
if (!docMigrator.isUpToDate) {
throw CliException(
"pkldoc website model is too old (found: ${docMigrator.docsiteVersion}, required: ${DocMigrator.CURRENT_VERSION}). Run `pkldoc --migrate` to migrate the website."
)
}
val docsiteInfoModuleUris = mutableListOf<URI>()
val packageInfoModuleUris = mutableListOf<URI>()
val regularModuleUris = mutableListOf<URI>()
@@ -271,8 +295,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
options.normalizedOutputDir,
options.isTestMode,
options.noSymlinks,
consoleOut,
docMigrator,
)
.run()
} catch (e: DocGeneratorBugException) {
throw CliBugException(e)
} catch (e: DocGeneratorException) {
throw CliException(e.message!!)
}

View File

@@ -47,6 +47,9 @@ constructor(
* however, will create a full copy instead.
*/
val noSymlinks: Boolean = false,
/** Migrate existing pkldoc */
val migrate: Boolean = false,
) {
/** [outputDir] after undergoing normalization. */
val normalizedOutputDir: Path = base.normalizedWorkingDir.resolveSafely(outputDir)

View File

@@ -16,9 +16,14 @@
package org.pkl.doc
import java.io.IOException
import java.io.OutputStream
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.util.concurrent.Executor
import java.util.concurrent.Executors
import kotlin.io.path.*
import kotlinx.coroutines.*
import org.pkl.commons.copyRecursively
import org.pkl.core.ModuleSchema
import org.pkl.core.PClassInfo
@@ -39,11 +44,11 @@ class DocGenerator(
*/
private val docsiteInfo: DocsiteInfo,
/** The modules to generate documentation for, grouped by package. */
modules: Map<DocPackageInfo, Collection<ModuleSchema>>,
/** The packages to generate documentation for. */
packages: Map<DocPackageInfo, Collection<ModuleSchema>>,
/**
* A function to resolve imports in [modules] and [docsiteInfo].
* A function to resolve imports in [packages] and [docsiteInfo].
*
* Module `pkl.base` is resolved with this function even if not explicitly imported.
*/
@@ -73,16 +78,26 @@ class DocGenerator(
* however, will create a full copy instead.
*/
private val noSymlinks: Boolean = false,
) {
/** The output stream to write logs to. */
consoleOut: OutputStream,
/**
* The doc migrator that is used to determine the latest docsite version, as well as update the
* version file.
*/
private val docMigrator: DocMigrator = DocMigrator(outputDir, consoleOut, versionComparator),
) : AbstractGenerator(consoleOut) {
companion object {
const val CURRENT_DIRECTORY_NAME = "current"
internal fun List<PackageData>.current(
versionComparator: Comparator<String>
internal fun determineCurrentPackages(
packages: List<PackageData>,
descendingVersionComparator: Comparator<String>,
): List<PackageData> {
val comparator =
compareBy<PackageData> { it.ref.pkg }.thenBy(versionComparator) { it.ref.version }
return asSequence()
compareBy<PackageData> { it.ref.pkg }.thenBy(descendingVersionComparator) { it.ref.version }
return packages
// If matching a semver pattern, remove any version that has a prerelease
// version (e.g. SNAPSHOT in 1.2.3-SNAPSHOT)
.filter { Version.parseOrNull(it.ref.version)?.preRelease == null }
@@ -90,50 +105,131 @@ class DocGenerator(
.distinctBy { it.ref.pkg }
.toList()
}
/**
* The default exeuctor when running the doc generator.
*
* Uses [Executors.newVirtualThreadPerTaskExecutor] if available (JDK 21). Otherwise, uses
* [Executors.newFixedThreadPool] with 64 threads, or the number of available processors
* available to the JVM (whichever is higher).
*/
internal val executor: Executor
get() {
try {
val method = Executors::class.java.getMethod("newVirtualThreadPerTaskExecutor")
return method.invoke(null) as Executor
} catch (e: Throwable) {
when (e) {
is NoSuchMethodException,
is IllegalAccessException ->
return Executors.newFixedThreadPool(
64.coerceAtLeast(Runtime.getRuntime().availableProcessors())
)
else -> throw e
}
}
}
}
private val descendingVersionComparator: Comparator<String> = versionComparator.reversed()
private val docPackages: List<DocPackage> = modules.map { DocPackage(it.key, it.value.toList()) }
private val docPackages: List<DocPackage> =
packages.map { DocPackage(it.key, it.value.toList()) }.filter { !it.isUnlisted }
private fun tryLoadPackageData(entry: SearchIndexGenerator.PackageIndexEntry): PackageData? {
var packageDataFile =
outputDir.resolve(entry.packageEntry.url).resolveSibling("package-data.json")
if (!Files.exists(packageDataFile)) {
// search-index.js in Pkl 0.29 and below did not encode path.
// If we get a file does not exist, try again by encoding the path.
packageDataFile =
outputDir.resolve(entry.packageEntry.url.pathEncoded).resolveSibling("package-data.json")
if (!Files.exists((packageDataFile))) {
writeOutputLine(
"[Warn] likely corrupted search index; missing $packageDataFile. This entry will be removed in the newly generated index."
)
return null
}
}
writeOutput("Loading package data for ${packageDataFile.toUri()}\r")
return PackageData.read(packageDataFile)
}
private suspend fun getCurrentPackages(
siteSearchIndex: List<SearchIndexGenerator.PackageIndexEntry>
): List<PackageData> = coroutineScope {
siteSearchIndex.map { async { tryLoadPackageData(it) } }.awaitAll().filterNotNull()
}
/** Runs this documentation generator. */
fun run() {
try {
val htmlGenerator =
HtmlGenerator(docsiteInfo, docPackages, importResolver, outputDir, isTestMode)
val searchIndexGenerator = SearchIndexGenerator(outputDir)
val packageDataGenerator = PackageDataGenerator(outputDir)
val runtimeDataGenerator = RuntimeDataGenerator(descendingVersionComparator, outputDir)
fun run() =
runBlocking(executor.asCoroutineDispatcher()) {
try {
if (!docMigrator.isUpToDate) {
throw DocGeneratorException(
"Docsite is not up to date. Expected: ${DocMigrator.CURRENT_VERSION}. Found: ${docMigrator.docsiteVersion}. Use DocMigrator to migrate the site."
)
}
val htmlGenerator =
HtmlGenerator(docsiteInfo, docPackages, importResolver, outputDir, isTestMode, consoleOut)
val searchIndexGenerator = SearchIndexGenerator(outputDir, consoleOut)
val packageDataGenerator = PackageDataGenerator(outputDir, consoleOut)
val runtimeDataGenerator =
RuntimeDataGenerator(descendingVersionComparator, outputDir, consoleOut)
for (docPackage in docPackages) {
if (docPackage.isUnlisted) continue
coroutineScope {
for (docPackage in docPackages) {
launch {
docPackage.deletePackageDir()
coroutineScope {
launch { htmlGenerator.generate(docPackage) }
launch { searchIndexGenerator.generate(docPackage) }
launch { packageDataGenerator.generate(docPackage) }
}
}
}
}
docPackage.deletePackageDir()
htmlGenerator.generate(docPackage)
searchIndexGenerator.generate(docPackage)
packageDataGenerator.generate(docPackage)
writeOutputLine("Generated HTML for packages")
val newlyGeneratedPackages = docPackages.map(::PackageData).sortedBy { it.ref.pkg }
val currentSearchIndex = searchIndexGenerator.getCurrentSearchIndex()
writeOutputLine("Loaded current search index")
val existingCurrentPackages = getCurrentPackages(currentSearchIndex)
writeOutputLine("Fetched latest packages")
val currentPackages =
determineCurrentPackages(
newlyGeneratedPackages + existingCurrentPackages,
descendingVersionComparator,
)
createCurrentDirectories(currentPackages, existingCurrentPackages)
searchIndexGenerator.generateSiteIndex(currentPackages)
htmlGenerator.generateSite(currentPackages)
runtimeDataGenerator.generate(newlyGeneratedPackages)
writeOutputLine("Wrote package runtime data files")
docMigrator.updateDocsiteVersion()
} catch (e: IOException) {
throw DocGeneratorBugException("I/O error generating documentation: $e", e)
}
val packagesData = packageDataGenerator.readAll()
val currentPackagesData = packagesData.current(descendingVersionComparator)
createCurrentDirectories(currentPackagesData)
htmlGenerator.generateSite(currentPackagesData)
searchIndexGenerator.generateSiteIndex(currentPackagesData)
runtimeDataGenerator.deleteDataDir()
runtimeDataGenerator.generate(packagesData)
} catch (e: IOException) {
throw DocGeneratorException("I/O error generating documentation.", e)
}
}
private fun DocPackage.deletePackageDir() {
outputDir.resolve(IoUtils.encodePath("$name/$version")).deleteRecursively()
}
private fun createCurrentDirectories(currentPackagesData: List<PackageData>) {
for (packageData in currentPackagesData) {
private fun createCurrentDirectories(
currentPackages: List<PackageData>,
existingCurrentPackages: List<PackageData>,
) {
val packagesToCreate = currentPackages - existingCurrentPackages
for (packageData in packagesToCreate) {
val basePath = outputDir.resolve(packageData.ref.pkg.pathEncoded)
val src = basePath.resolve(packageData.ref.version)
val dst = basePath.resolve(CURRENT_DIRECTORY_NAME)
@@ -218,7 +314,7 @@ internal class DocModule(
get() = schema.moduleName
val path: String by lazy {
name.substring(parent.docPackageInfo.moduleNamePrefix.length).replace('.', '/')
name.pathEncoded.substring(parent.docPackageInfo.moduleNamePrefix.length).replace('.', '/')
}
val overview: String?

View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,5 +15,8 @@
*/
package org.pkl.doc
class DocGeneratorException(message: String, cause: Throwable? = null) :
open class DocGeneratorException(message: String, cause: Throwable? = null) :
RuntimeException(message, cause)
class DocGeneratorBugException(message: String, cause: Throwable? = null) :
DocGeneratorException(message, cause)

View File

@@ -0,0 +1,279 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.doc
import java.io.OutputStream
import java.nio.file.Files
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import java.util.concurrent.atomic.AtomicInteger
import kotlin.io.path.ExperimentalPathApi
import kotlin.io.path.createParentDirectories
import kotlin.io.path.deleteIfExists
import kotlin.io.path.exists
import kotlin.io.path.isRegularFile
import kotlin.io.path.listDirectoryEntries
import kotlin.io.path.name
import kotlin.io.path.readLines
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.json.Json
import org.pkl.commons.lazyWithReceiver
import org.pkl.commons.readString
import org.pkl.commons.toUri
import org.pkl.commons.writeString
import org.pkl.core.util.IoUtils
import org.pkl.doc.RuntimeData.Companion.EMPTY
/** Migrates an existing Pkldoc site from version 1 to version 2. */
@OptIn(ExperimentalPathApi::class)
class DocMigrator(
private val outputDir: Path,
consoleOut: OutputStream,
versionComparator: Comparator<String>,
) : AbstractGenerator(consoleOut) {
companion object {
const val CURRENT_VERSION = 2
private val json = Json {
ignoreUnknownKeys = true
explicitNulls = false
}
private const val LEGACY_KNOWN_VERSIONS_PREFIX =
"runtimeData.links('${HtmlConstants.KNOWN_VERSIONS}','"
private const val LEGACY_KNOWN_USAGES_PREFIX =
"runtimeData.links('${HtmlConstants.KNOWN_USAGES}','"
private const val LEGACY_KNOWN_SUBTYPES_PREFIX =
"runtimeData.links('${HtmlConstants.KNOWN_SUBTYPES}','"
private const val LEGACY_SUFFIX = "');"
internal fun parseLegacyRuntimeData(path: Path, myVersionHref: String): RuntimeData {
try {
var runtimeData = EMPTY
for (line in Files.lines(path)) {
runtimeData =
when {
line.startsWith(LEGACY_KNOWN_VERSIONS_PREFIX) -> {
val knownVersions =
readLegacyLine(line, LEGACY_KNOWN_VERSIONS_PREFIX, LEGACY_SUFFIX).mapTo(
mutableSetOf()
) { link ->
// fill in missing href
if (link.href != null) link else link.copy(href = myVersionHref)
}
runtimeData.copy(knownVersions = knownVersions)
}
line.startsWith(LEGACY_KNOWN_USAGES_PREFIX) -> {
val knownUsages = readLegacyLine(line, LEGACY_KNOWN_USAGES_PREFIX, LEGACY_SUFFIX)
runtimeData.copy(knownUsages = knownUsages)
}
line.startsWith(LEGACY_KNOWN_SUBTYPES_PREFIX) -> {
val knownSubtypes =
readLegacyLine(line, LEGACY_KNOWN_SUBTYPES_PREFIX, LEGACY_SUFFIX)
runtimeData.copy(knownSubtypes = knownSubtypes)
}
else -> throw RuntimeException("Unexpected runtimeData line: $line")
}
}
return runtimeData
} catch (e: NoSuchFileException) {
throw e
}
}
private fun readLegacyLine(line: String, prefix: String, suffix: String): Set<RuntimeDataLink> {
val jsStr = line.substring(prefix.length, line.length - suffix.length)
return json.decodeFromString<List<RuntimeDataLink>>(jsStr).toSet()
}
}
val isUpToDate by lazy {
if (!Files.exists(outputDir.resolve("index.html"))) {
// must be the first run
return@lazy true
}
docsiteVersion == CURRENT_VERSION
}
val docsiteVersion by lazy {
if (!versionPath.isRegularFile()) 1 else versionPath.readString().trim().toInt()
}
private val versionPath
get() = outputDir.resolve(".pkldoc/VERSION")
private val descendingVersionComparator = versionComparator.reversed()
fun updateDocsiteVersion() {
versionPath.createParentDirectories()
versionPath.writeString(CURRENT_VERSION.toString())
}
private suspend fun migratePackage(pkgData: PackageData, isCurrentVersion: Boolean) {
coroutineScope {
if (!isCurrentVersion) {
launch { migrateRuntimeData(pkgData) }
}
launch { migrateHtml(pkgData, isCurrentVersion) }
}
}
fun run() =
runBlocking(Dispatchers.IO) {
if (isUpToDate) {
consoleOut.writeLine(
"Generated pkldoc is already version $CURRENT_VERSION; there's nothing to do."
)
return@runBlocking
}
val packageDatas = Files.walk(outputDir).filter { it.name == "package-data.json" }.toList()
val count = AtomicInteger(1)
for (path in packageDatas) {
val pkgData = PackageData.read(path)
val isCurrentVersion = path.parent.name == "current"
migratePackage(pkgData, isCurrentVersion)
if (!isCurrentVersion) {
deleteLegacyRuntimeData(pkgData)
}
consoleOut.write(
"Migrated ${count.incrementAndGet()} packages (${pkgData.ref.pkg}@${pkgData.ref.version})\r"
)
consoleOut.flush()
}
launch { copyResource("scripts/pkldoc.js", outputDir) }
updateDocsiteVersion()
consoleOut.writeLine("Finished migration, migrated $count packages.")
}
private suspend fun migrateHtml(packageData: PackageData, isCurrentVersion: Boolean) =
coroutineScope {
for (ref in packageData.refs) {
launch { doMigrateHtml(ref, outputDir.resolveHtmlPath(ref, isCurrentVersion)) }
}
}
private fun Path.resolveHtmlPath(ref: ElementRef<*>, isCurrentVersion: Boolean): Path {
val effectiveRef = if (isCurrentVersion) ref.withVersion("current") else ref
return resolve(effectiveRef.htmlPath)
}
private val migratedCurrentPackages = mutableSetOf<String>()
private fun doMigrateHtml(ref: ElementRef<*>, path: Path) {
val newHtml = buildString {
val lines = path.readLines()
for ((idx, line) in lines.withIndex()) {
if (line.contains(ref.legacyVersionedRuntimeDataPath)) continue
appendLine(line)
if (line.contains("scripts/pkldoc.js")) {
if (lines.getOrNull(idx + 1)?.contains("<script type=\"module\"") == false) {
val packageVersionUrl =
IoUtils.relativize(ref.perPackageRuntimeDataPath.toUri(), ref.htmlPath.toUri())
.toString()
val perPackageVersionUrl =
IoUtils.relativize(ref.perPackageVersionRuntimeDataPath.toUri(), ref.htmlPath.toUri())
.toString()
appendLine(
""" <script type="module">
import perPackageData from "$packageVersionUrl" with { type: "json" }
import perPackageVersionData from "$perPackageVersionUrl" with { type: "json" }
runtimeData.knownVersions(perPackageData.knownVersions, ${json.encodeToString(ref.version)});
runtimeData.knownUsagesOrSubtypes("known-subtypes", perPackageVersionData.knownSubtypes);
runtimeData.knownUsagesOrSubtypes("known-usages", perPackageVersionData.knownUsages);
</script>"""
)
}
}
}
}
path.writeString(newHtml)
}
private fun getLatestPackageData(pkg: String): PackageData {
val currentPackage = outputDir.resolve("${pkg.pathEncoded}/current/package-data.json")
if (currentPackage.isRegularFile()) {
return PackageData.read(currentPackage)
}
// it's possible that the "current" path doesn't exist if there are only pre-releases.
val versions = currentPackage.parent.parent.listDirectoryEntries()
val latestVersion = versions.map { it.name }.sortedWith(descendingVersionComparator).first()
return PackageData.read(
outputDir.resolve("${pkg.pathEncoded}/$latestVersion/package-data.json")
)
}
/** Convert legacy style data paths to new style paths */
private suspend fun migrateRuntimeData(packageData: PackageData) = coroutineScope {
if (!migratedCurrentPackages.contains(packageData.ref.pkg)) {
val currentPackageData = getLatestPackageData(packageData.ref.pkg)
for (ref in currentPackageData.refs) {
launch { doMigratePerPackageData(ref) }
}
migratedCurrentPackages.add(packageData.ref.pkg)
}
for (ref in packageData.refs) {
launch { doMigratePerPackageVersionData(ref) }
}
}
private val PackageData.refs
get(): List<ElementRef<*>> {
return buildList {
add(ref)
for (module in modules) {
add(module.ref)
for (clazz in module.classes) {
add(clazz.ref)
}
}
}
}
private val ElementRef<*>.legacyRuntimeData: RuntimeData? by lazyWithReceiver {
val legacyPath = outputDir.resolve(legacyVersionedRuntimeDataPath)
when {
legacyPath.exists() ->
parseLegacyRuntimeData(legacyPath, myVersionHref = pageUrlForVersion(version))
else -> null
}
}
private fun doMigratePerPackageVersionData(ref: ElementRef<*>) {
val data = ref.legacyRuntimeData ?: return
data.perPackageVersion().writeTo(outputDir.resolve(ref.perPackageVersionRuntimeDataPath))
}
private fun doMigratePerPackageData(ref: ElementRef<*>) {
val data = ref.legacyRuntimeData ?: return
data.perPackage().writeTo(outputDir.resolve(ref.perPackageRuntimeDataPath))
}
private suspend fun deleteLegacyRuntimeData(packageData: PackageData) = coroutineScope {
for (ref in packageData.refs) {
launch { outputDir.resolve(ref.legacyVersionedRuntimeDataPath).deleteIfExists() }
}
}
}

View File

@@ -117,7 +117,10 @@ internal sealed class DocScope {
/** A scope that corresponds to an entire Pkldoc page. */
internal abstract class PageScope : DocScope() {
/** The location of the runtime data file for this page. */
abstract val dataUrl: URI
abstract val perPackageDataUrl: URI
/** The location of the runtime data JSON file that is per-version for this page. */
abstract val perPackageVersionDataUrl: URI
}
// equality is identity
@@ -125,15 +128,15 @@ internal class SiteScope(
val docPackages: List<DocPackage>,
private val overviewImports: Map<String, URI>,
private val importResolver: (URI) -> ModuleSchema,
outputDir: Path,
val outputDir: Path,
) : PageScope() {
private val pklVersion = Release.current().version().withBuild(null).toString()
private val pklBaseModule: ModuleSchema by lazy { importResolver("pkl:base".toUri()) }
val packageScopes: Map<String, PackageScope> by lazy {
val packageScopes: Map<DocPackageInfo, PackageScope> by lazy {
docPackages.associate { docPackage ->
docPackage.name to
docPackage.docPackageInfo to
PackageScope(
docPackage.docPackageInfo,
docPackage.docModules.map { it.schema },
@@ -144,6 +147,11 @@ internal class SiteScope(
}
}
val stdlibPackageScope: PackageScope? by lazy {
val pklPackage = docPackages.find { it.name == "pkl" }
packageScopes[pklPackage?.docPackageInfo]
}
private val pklBaseScope: ModuleScope by lazy {
ModuleScope(pklBaseModule, resolveModuleNameToDocUrl("pkl.base")!!, null)
}
@@ -155,7 +163,10 @@ internal class SiteScope(
IoUtils.ensurePathEndsWithSlash(outputDir.toUri()).resolve("index.html")
}
override val dataUrl: URI
override val perPackageDataUrl: URI
get() = throw UnsupportedOperationException("perVersionDataUrl")
override val perPackageVersionDataUrl: URI
get() = throw UnsupportedOperationException("perVersionDataUrl")
fun createEmptyPackageScope(
@@ -187,7 +198,7 @@ internal class SiteScope(
override fun getProperty(name: String): DocScope? = null
fun getPackage(name: String): PackageScope = packageScopes.getValue(name)
fun getPackage(packageInfo: DocPackageInfo): PackageScope = packageScopes.getValue(packageInfo)
fun resolveImport(uri: URI): ModuleSchema = importResolver(uri)
@@ -195,7 +206,7 @@ internal class SiteScope(
when {
name.startsWith("pkl.") -> {
val packagePage =
packageScopes["pkl"]?.url // link to locally generated stdlib docs if available
stdlibPackageScope?.url // link to locally generated stdlib docs if available
?: PklInfo.current().packageIndex.getPackagePage("pkl", pklVersion).toUri()
packagePage.resolve(name.substring(4) + "/")
}
@@ -264,8 +275,12 @@ internal class PackageScope(
return IoUtils.relativize(myVersion, scope.url)
}
override val dataUrl: URI by lazy {
parent.url.resolve("./data/${name.pathEncoded}/$version/index.js")
override val perPackageDataUrl: URI by lazy {
parent.url.resolve("./data/${name.pathEncoded}/_/index.json")
}
override val perPackageVersionDataUrl: URI by lazy {
parent.url.resolve("./data/${name.pathEncoded}/$version/index.json")
}
fun getModule(name: String): ModuleScope = moduleScopes.getValue(name)
@@ -330,7 +345,13 @@ internal class ModuleScope(
getModulePath(module.moduleName, parent!!.docPackageInfo.moduleNamePrefix).uriEncoded
}
override val dataUrl: URI by lazy { parent!!.dataUrl.resolve("./$path/index.js") }
override val perPackageDataUrl: URI by lazy {
parent!!.perPackageDataUrl.resolve("./$path/index.json")
}
override val perPackageVersionDataUrl: URI by lazy {
parent!!.perPackageVersionDataUrl.resolve("./$path/index.json")
}
override fun getMethod(name: String): MethodScope? =
module.moduleClass.allMethods[name]?.let { MethodScope(it, this) }
@@ -393,8 +414,14 @@ internal class ClassScope(
else parentUrl.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.html")
}
override val dataUrl: URI by lazy {
parent!!.dataUrl.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.js")
override val perPackageDataUrl: URI by lazy {
parent!!.perPackageDataUrl.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.json")
}
override val perPackageVersionDataUrl: URI by lazy {
parent!!
.perPackageVersionDataUrl
.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.json")
}
override fun getMethod(name: String): MethodScope? =

View File

@@ -15,8 +15,11 @@
*/
package org.pkl.doc
import java.io.OutputStream
import java.net.URI
import java.nio.file.Path
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import org.pkl.core.ModuleSchema
internal class HtmlGenerator(
@@ -25,67 +28,70 @@ internal class HtmlGenerator(
importResolver: (URI) -> ModuleSchema,
private val outputDir: Path,
private val isTestMode: Boolean,
) {
consoleOut: OutputStream,
) : AbstractGenerator(consoleOut) {
private val siteScope =
SiteScope(docPackages, docsiteInfo.overviewImports, importResolver, outputDir)
fun generate(docPackage: DocPackage) {
val packageScope = siteScope.getPackage(docPackage.name)
PackagePageGenerator(docsiteInfo, docPackage, packageScope).run()
suspend fun generate(docPackage: DocPackage) = coroutineScope {
val packageScope = siteScope.getPackage(docPackage.docPackageInfo)
launch { PackagePageGenerator(docsiteInfo, docPackage, packageScope, consoleOut).run() }
for (docModule in docPackage.docModules) {
if (docModule.isUnlisted) continue
val moduleScope = packageScope.getModule(docModule.name)
ModulePageGenerator(docsiteInfo, docPackage, docModule, moduleScope, isTestMode).run()
launch {
ModulePageGenerator(docsiteInfo, docPackage, docModule, moduleScope, isTestMode, consoleOut)
.run()
}
for ((_, clazz) in docModule.schema.classes) {
if (clazz.isUnlisted) continue
ClassPageGenerator(
docsiteInfo,
docPackage,
docModule,
clazz,
ClassScope(clazz, moduleScope.url, moduleScope),
isTestMode,
)
.run()
launch {
ClassPageGenerator(
docsiteInfo,
docPackage,
docModule,
clazz,
ClassScope(clazz, moduleScope.url, moduleScope),
isTestMode,
consoleOut,
)
.run()
}
}
}
}
fun generateSite(packagesData: List<PackageData>) {
MainPageGenerator(docsiteInfo, packagesData, siteScope).run()
generateStaticResources()
suspend fun generateSite(packages: List<PackageData>) = coroutineScope {
launch { MainPageGenerator(docsiteInfo, packages, siteScope, consoleOut).run() }
launch { generateStaticResources() }
}
private fun generateStaticResources() {
copyResource("fonts/lato-v14-latin_latin-ext-regular.woff2", outputDir)
copyResource("fonts/lato-v14-latin_latin-ext-700.woff2", outputDir)
copyResource("fonts/open-sans-v15-latin_latin-ext-regular.woff2", outputDir)
copyResource("fonts/open-sans-v15-latin_latin-ext-italic.woff2", outputDir)
copyResource("fonts/open-sans-v15-latin_latin-ext-700.woff2", outputDir)
copyResource("fonts/open-sans-v15-latin_latin-ext-700italic.woff2", outputDir)
copyResource("fonts/source-code-pro-v7-latin_latin-ext-regular.woff2", outputDir)
copyResource("fonts/source-code-pro-v7-latin_latin-ext-700.woff2", outputDir)
copyResource("fonts/MaterialIcons-Regular.woff2", outputDir)
copyResource("scripts/pkldoc.js", outputDir)
copyResource("scripts/search-worker.js", outputDir)
copyResource("scripts/scroll-into-view.min.js", outputDir)
copyResource("styles/pkldoc.css", outputDir)
copyResource("images/apple-touch-icon.png", outputDir)
copyResource("images/favicon.svg", outputDir)
copyResource("images/favicon-16x16.png", outputDir)
copyResource("images/favicon-32x32.png", outputDir)
private suspend fun generateStaticResources() = coroutineScope {
val resources =
listOf(
"fonts/lato-v14-latin_latin-ext-regular.woff2",
"fonts/lato-v14-latin_latin-ext-700.woff2",
"fonts/open-sans-v15-latin_latin-ext-regular.woff2",
"fonts/open-sans-v15-latin_latin-ext-italic.woff2",
"fonts/open-sans-v15-latin_latin-ext-700.woff2",
"fonts/open-sans-v15-latin_latin-ext-700italic.woff2",
"fonts/source-code-pro-v7-latin_latin-ext-regular.woff2",
"fonts/source-code-pro-v7-latin_latin-ext-700.woff2",
"fonts/MaterialIcons-Regular.woff2",
"scripts/pkldoc.js",
"scripts/search-worker.js",
"scripts/scroll-into-view.min.js",
"styles/pkldoc.css",
"images/apple-touch-icon.png",
"images/favicon.svg",
"images/favicon-16x16.png",
"images/favicon-32x32.png",
)
for (resource in resources) {
launch { copyResource(resource, outputDir) }
}
}
}

View File

@@ -50,7 +50,7 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
help = "Module paths/uris, or package uris to generate documentation for",
)
.convert { parseModuleName(it) }
.multiple(required = true)
.multiple()
private val outputDir: Path by
option(
@@ -72,6 +72,14 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
private val isTestMode by
option(names = arrayOf("--test-mode"), help = "Internal test mode", hidden = true).flag()
private val migrate: Boolean by
option(
names = arrayOf("--migrate"),
help = "Migrate a pkl-doc site from version 0 to version 1.",
)
.single()
.flag(default = false)
private val projectOptions by ProjectOptions()
override val helpString: String = "Generate HTML documentation from Pkl modules and packages."
@@ -83,6 +91,7 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
outputDir,
isTestMode,
noSymlinks,
migrate,
)
CliDocGenerator(options).run()
}

View File

@@ -15,13 +15,15 @@
*/
package org.pkl.doc
import java.io.OutputStream
import kotlinx.html.*
internal abstract class MainOrPackagePageGenerator<S>(
docsiteInfo: DocsiteInfo,
pageScope: S,
private val siteScope: SiteScope,
) : PageGenerator<S>(docsiteInfo, pageScope) where S : PageScope {
consoleOut: OutputStream,
) : PageGenerator<S>(docsiteInfo, pageScope, consoleOut) where S : PageScope {
protected fun UL.renderModuleOrPackage(
name: String,
moduleOrPackageScope: DocScope,

View File

@@ -15,13 +15,15 @@
*/
package org.pkl.doc
import java.io.OutputStream
import kotlinx.html.*
internal class MainPageGenerator(
docsiteInfo: DocsiteInfo,
private val packagesData: List<PackageData>,
pageScope: SiteScope,
) : MainOrPackagePageGenerator<SiteScope>(docsiteInfo, pageScope, pageScope) {
consoleOut: OutputStream,
) : MainOrPackagePageGenerator<SiteScope>(docsiteInfo, pageScope, pageScope, consoleOut) {
override val html: HTML.() -> Unit = {
renderHtmlHead()
@@ -89,14 +91,12 @@ internal class MainPageGenerator(
ul {
for (pkg in sortedPackages) {
val packageScope =
pageScope.packageScopes[pkg.ref.pkg]
// create scope for previously generated package
?: pageScope.createEmptyPackageScope(
pkg.ref.pkg,
pkg.ref.version,
pkg.sourceCodeUrlScheme,
pkg.sourceCode,
)
pageScope.createEmptyPackageScope(
pkg.ref.pkg,
pkg.ref.version,
pkg.sourceCodeUrlScheme,
pkg.sourceCode,
)
val memberDocs =
MemberDocs(

View File

@@ -15,6 +15,7 @@
*/
package org.pkl.doc
import java.io.OutputStream
import java.io.StringWriter
import kotlinx.html.*
import org.pkl.core.Member
@@ -31,7 +32,8 @@ internal abstract class ModuleOrClassPageGenerator<S>(
protected val clazz: PClass,
scope: S,
private val isTestMode: Boolean,
) : PageGenerator<S>(docsiteInfo, scope) where S : PageScope {
consoleOut: OutputStream,
) : PageGenerator<S>(docsiteInfo, scope, consoleOut) where S : PageScope {
protected fun HtmlBlockTag.renderProperties() {
if (!clazz.hasListedProperty) return

View File

@@ -15,6 +15,7 @@
*/
package org.pkl.doc
import java.io.OutputStream
import kotlinx.html.*
internal class ModulePageGenerator(
@@ -23,6 +24,7 @@ internal class ModulePageGenerator(
docModule: DocModule,
pageScope: ModuleScope,
isTestMode: Boolean,
consoleOut: OutputStream,
) :
ModuleOrClassPageGenerator<ModuleScope>(
docsiteInfo,
@@ -30,6 +32,7 @@ internal class ModulePageGenerator(
docModule.schema.moduleClass,
pageScope,
isTestMode,
consoleOut,
) {
private val module = docModule.schema

View File

@@ -16,6 +16,7 @@
package org.pkl.doc
import java.io.IOException
import java.io.OutputStream
import java.net.URI
import java.nio.file.Path
import kotlin.io.path.createParentDirectories
@@ -24,8 +25,8 @@ import kotlinx.serialization.*
import kotlinx.serialization.json.Json
import org.pkl.commons.readString
import org.pkl.commons.toUri
import org.pkl.commons.walk
import org.pkl.core.*
import org.pkl.core.packages.PackageUri
import org.pkl.core.util.IoUtils
/**
@@ -33,7 +34,8 @@ import org.pkl.core.util.IoUtils
* previously generated docs in a newly generated doc website. This is useful if there's a problem
* with fetching or evaluating the latest package version.
*/
internal class PackageDataGenerator(private val outputDir: Path) {
internal class PackageDataGenerator(private val outputDir: Path, consoleOut: OutputStream) :
AbstractGenerator(consoleOut) {
fun generate(pkg: DocPackage) {
val path =
outputDir
@@ -43,19 +45,10 @@ internal class PackageDataGenerator(private val outputDir: Path) {
.apply { createParentDirectories() }
PackageData(pkg).write(path)
}
fun readAll(): List<PackageData> {
return outputDir.walk().use { paths ->
paths
.filter { it.fileName?.toString() == "package-data.json" }
.map { PackageData.read(it) }
.toList()
}
}
}
/** Uniquely identifies a specific version of a package, module, class, or type alias. */
internal sealed class ElementRef {
internal sealed class ElementRef<T : ElementRef<T>> {
/** The package name. */
abstract val pkg: String
@@ -66,15 +59,51 @@ internal sealed class ElementRef {
abstract val version: String
/** The Pkldoc page URL of the element relative to its Pkldoc website root. */
abstract val pageUrl: URI
val htmlPath: String by lazy { "$basePath/${version.pathEncoded}/$packageRelativeHtmlPath" }
/** The Pkldoc runtime data JSON path for this ref. */
val perPackageRuntimeDataPath: String by lazy { "data/$basePath/_/$packageRelativeDataPath" }
/** The Pkldoc runtime data JSON path for this ref at a per-version level. */
val perPackageVersionRuntimeDataPath: String by lazy {
"data/$basePath/${version.pathEncoded}/$packageRelativeDataPath"
}
/** The legacy runtime data path () */
val legacyVersionedRuntimeDataPath: String by lazy {
"data/$basePath/${version.pathEncoded}/${packageRelativeDataPath.substringBeforeLast(".json") + ".js"}"
}
/**
* The Pkldoc page URL of the element relative to [other]'s page URL. Assumes that both elements
* have the same Pkldoc website root.
*/
fun pageUrlRelativeTo(other: ElementRef): String {
return IoUtils.relativize(pageUrl, other.pageUrl).toString()
fun pageUrlRelativeTo(other: ElementRef<*>): String {
return IoUtils.relativize(htmlPath.toUri(), other.htmlPath.toUri()).toString()
}
/** The Pkldoc page URL for my element with a different veresion. */
fun pageUrlForVersion(version: String): String {
val pathToBasePath = IoUtils.relativize(basePath.toUri(), htmlPath.toUri()).toString()
return "${pathToBasePath}${version.pathEncoded}/${withVersion(version).packageRelativeHtmlPath}"
}
abstract fun withVersion(version: String): T
/** The HTML path within the package folder */
abstract val packageRelativeHtmlPath: String
/** The JSON path within the package folder */
abstract val packageRelativeDataPath: String
fun isInSamePackageAs(other: ElementRef<T>): Boolean =
pkg == other.pkg && pkgUri == other.pkgUri && version == other.version
val basePath: String
get() =
if (pkgUri?.scheme == "package")
"${pkgUri!!.authority}${PackageUri(pkgUri!!).pathWithoutVersion}".pathEncoded
else pkg.pathEncoded
}
/** Uniquely identifies a specific version of a package. */
@@ -88,8 +117,12 @@ internal data class PackageRef(
/** The package version. */
override val version: String,
) : ElementRef() {
override val pageUrl: URI by lazy { "$pkg/$version/index.html".toUri() }
) : ElementRef<PackageRef>() {
override val packageRelativeHtmlPath: String = "index.html"
override val packageRelativeDataPath: String = "index.json"
override fun withVersion(version: String): PackageRef = copy(version = version)
}
/** Uniquely identifies a specific version of a module. */
@@ -106,12 +139,13 @@ internal data class ModuleRef(
/** The module path. */
val module: String,
) : ElementRef() {
override val pageUrl: URI by lazy { "$pkg/$version/$module/index.html".toUri() }
) : ElementRef<ModuleRef>() {
override val packageRelativeHtmlPath: String by lazy { "${module.pathEncoded}/index.html" }
val moduleClassRef: TypeRef by lazy {
TypeRef(pkg, pkgUri, version, module, PClassInfo.MODULE_CLASS_NAME)
}
override val packageRelativeDataPath: String
get() = "$module/index.json".pathEncoded
override fun withVersion(version: String): ModuleRef = copy(version = version)
val id: ModuleId by lazy { ModuleId(pkg, module) }
@@ -138,20 +172,32 @@ internal data class TypeRef(
/** Whether this is a type alias rather than a class. */
val isTypeAlias: Boolean = false,
) : ElementRef() {
) : ElementRef<TypeRef>() {
val id: TypeId by lazy { TypeId(pkg, module, type) }
val displayName: String by lazy { if (isModuleClass) module.substringAfterLast('/') else type }
override val pageUrl: URI by lazy {
override val packageRelativeHtmlPath: String by lazy {
when {
isTypeAlias -> "$pkg/$version/$module/index.html#$type".toUri()
isModuleClass -> "$pkg/$version/$module/index.html".toUri()
else -> "$pkg/$version/$module/$type.html".toUri()
isTypeAlias -> "$module/index.html#$type".pathEncoded
isModuleClass -> "$module/index.html".pathEncoded
else -> "$module/$type.html".pathEncoded
}
}
override val packageRelativeDataPath: String
get() {
// typealiases don't have their own runtime data.
require(!isTypeAlias) { "typealiases don't have runtime data" }
return when {
isModuleClass -> "$module/index.json".pathEncoded
else -> "$module/$type.json".pathEncoded
}
}
override fun withVersion(version: String): TypeRef = copy(version = version)
private val isModuleClass: Boolean
get() = type == PClassInfo.MODULE_CLASS_NAME
}
@@ -200,13 +246,13 @@ internal class PackageData(
try {
path.readString()
} catch (e: IOException) {
throw DocGeneratorException("I/O error reading `$path`.", e)
throw DocGeneratorBugException("I/O error reading `${path.toUri()}`.", e)
}
return try {
json.decodeFromString(jsonStr)
} catch (e: SerializationException) {
throw DocGeneratorException("Error deserializing `$path`.", e)
throw DocGeneratorBugException("Error deserializing `${path.toUri()}`.", e)
}
}
}
@@ -228,14 +274,14 @@ internal class PackageData(
try {
json.encodeToString(this)
} catch (e: SerializationException) {
throw DocGeneratorException("Error serializing `$path`.", e)
throw DocGeneratorBugException("Error serializing `$path`.", e)
}
try {
path.createParentDirectories()
path.writer().use { it.write(jsonStr) }
} catch (e: IOException) {
throw DocGeneratorException("I/O error writing `$path`.", e)
throw DocGeneratorBugException("I/O error writing `$path`.", e)
}
}
}

View File

@@ -15,13 +15,15 @@
*/
package org.pkl.doc
import java.io.OutputStream
import kotlinx.html.*
internal class PackagePageGenerator(
docsiteInfo: DocsiteInfo,
private val docPackage: DocPackage,
pageScope: PackageScope,
) : MainOrPackagePageGenerator<PackageScope>(docsiteInfo, pageScope, pageScope.parent) {
consoleOut: OutputStream,
) : MainOrPackagePageGenerator<PackageScope>(docsiteInfo, pageScope, pageScope.parent, consoleOut) {
override val html: HTML.() -> Unit = {
renderHtmlHead()

View File

@@ -15,10 +15,13 @@
*/
package org.pkl.doc
import java.io.OutputStream
import kotlin.io.path.bufferedWriter
import kotlin.io.path.createParentDirectories
import kotlin.io.path.exists
import kotlinx.html.*
import kotlinx.html.stream.appendHTML
import kotlinx.serialization.json.Json
import org.commonmark.ext.gfm.tables.TablesExtension
import org.commonmark.parser.Parser
import org.commonmark.renderer.html.HtmlRenderer
@@ -29,7 +32,12 @@ import org.pkl.core.util.IoUtils
internal abstract class PageGenerator<out S>(
protected val docsiteInfo: DocsiteInfo,
protected val pageScope: S,
) where S : PageScope {
consoleOut: OutputStream,
) : AbstractGenerator(consoleOut) where S : PageScope {
companion object {
private val json = Json {}
}
private val markdownInlineParserFactory = MarkdownParserFactory(pageScope)
private val markdownParser =
@@ -47,33 +55,62 @@ internal abstract class PageGenerator<out S>(
fun run() {
val path = pageScope.url.toPath()
path.createParentDirectories()
path.bufferedWriter().use {
it.appendLine("<!DOCTYPE html>")
it.appendHTML().html(null, html)
path.bufferedWriter().use { writer ->
writer.appendLine("<!DOCTYPE html>")
writer.appendHTML().html(null, html)
}
writeOutput("Wrote file ${pageScope.url}\r")
}
protected abstract val html: HTML.() -> Unit
protected abstract fun HTMLTag.renderPageTitle()
private fun DocScope?.getVersion(): String? {
return when (this) {
null -> null
is SiteScope -> null
is PackageScope -> version
else -> parent?.getVersion()
}
}
protected fun HTML.renderHtmlHead() {
lang = "en-US"
head {
meta { charset = "UTF-8" }
title { renderPageTitle() }
script {
src = pageScope.relativeSiteUrl.resolve("scripts/pkldoc.js").toString()
defer = true
}
pageScope.getVersion()?.let { version ->
script {
type = "module"
unsafe {
val packageVersionUrl =
IoUtils.relativize(pageScope.perPackageDataUrl, pageScope.url).toString()
val perPackageVersionUrl =
IoUtils.relativize(pageScope.perPackageVersionDataUrl, pageScope.url).toString()
raw(
"""
import perPackageData from "$packageVersionUrl" with { type: "json" }
import perPackageVersionData from "$perPackageVersionUrl" with { type: "json" }
runtimeData.knownVersions(perPackageData.knownVersions, ${json.encodeToString(version)});
runtimeData.knownUsagesOrSubtypes("known-subtypes", perPackageVersionData.knownSubtypes);
runtimeData.knownUsagesOrSubtypes("known-usages", perPackageVersionData.knownUsages);
"""
)
}
}
}
script {
src = pageScope.relativeSiteUrl.resolve("scripts/scroll-into-view.min.js").toString()
defer = true
}
if (pageScope !is SiteScope) {
script {
src = IoUtils.relativize(pageScope.dataUrl, pageScope.url).toString()
defer = true
}
}
link {
href = pageScope.relativeSiteUrl.resolve("styles/pkldoc.css").toString()
media = "screen"
@@ -102,7 +139,6 @@ internal abstract class PageGenerator<out S>(
sizes = "16x16"
href = pageScope.relativeSiteUrl.resolve("images/favicon-16x16.png").toString()
}
meta { charset = "UTF-8" }
}
}
@@ -127,10 +163,13 @@ internal abstract class PageGenerator<out S>(
div {
id = "search"
i {
id = "search-icon"
classes = setOf("material-icons")
+"search"
label {
htmlFor = "search-input"
i {
id = "search-icon"
classes = setOf("material-icons")
+"search"
}
}
input {
@@ -562,7 +601,25 @@ internal abstract class PageGenerator<out S>(
return siteScope.packageScopes.values
.find { it.name == dep.qualifiedName }
?.urlForVersionRelativeTo(pageScope, dep.version)
?.toString()
?.toString() ?: findAlreadyExistingPackageScope(siteScope, dep.qualifiedName, dep.version)
}
private fun findAlreadyExistingPackageScope(
siteScope: SiteScope,
name: String,
version: String,
): String? {
// hack: determine if we know about this dependency just by checking to see if the directory
// exists.
// we don't need to know if the specific _version_ exists (we assume that the version must be
// published by the docsite and tolerate possibly broken links).
val pkgPath = siteScope.outputDir.resolve(name.pathEncoded)
if (pkgPath.exists()) {
val targetPath = pkgPath.resolve("$version/index.html")
val myUrl = pageScope.url
return IoUtils.relativize(targetPath.toUri(), myUrl).toString()
}
return null
}
protected class MemberInfoKey(val name: String, val classes: Set<String> = setOf())
@@ -619,12 +676,12 @@ internal abstract class PageGenerator<out S>(
}
}
result[MemberInfoKey("Known subtypes", runtimeDataClasses)] = {
result[MemberInfoKey("Known subtypes in package", runtimeDataClasses)] = {
id = HtmlConstants.KNOWN_SUBTYPES
classes = runtimeDataClasses
}
result[MemberInfoKey("Known usages", runtimeDataClasses)] = {
result[MemberInfoKey("Known usages in package", runtimeDataClasses)] = {
id = HtmlConstants.KNOWN_USAGES
classes = runtimeDataClasses
}

View File

@@ -0,0 +1,160 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.doc
import java.io.FileNotFoundException
import java.nio.file.NoSuchFileException
import java.nio.file.Path
import kotlin.io.path.createParentDirectories
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.Serializable
import kotlinx.serialization.SerializationException
import kotlinx.serialization.json.Json
import org.pkl.commons.readString
import org.pkl.commons.writeString
@Serializable internal data class RuntimeDataLink(val text: String, val href: String?)
@OptIn(ExperimentalSerializationApi::class)
@Serializable
internal data class RuntimeData(
val knownVersions: Set<RuntimeDataLink> = setOf(),
val knownUsages: Set<RuntimeDataLink> = setOf(),
val knownSubtypes: Set<RuntimeDataLink> = setOf(),
) {
companion object {
val EMPTY = RuntimeData()
private val json = Json {
prettyPrint = true
explicitNulls = false
ignoreUnknownKeys = true
prettyPrintIndent = " "
}
/** Compare two paths, comparing each segment. */
private fun segmentComparator(comparator: Comparator<String>): Comparator<RuntimeDataLink> {
return Comparator { o1, o2 ->
val path1Segments = o1.href!!.split("/")
val path2Segments = o2.href!!.split("/")
for ((path1, path2) in path1Segments.zip(path2Segments)) {
if (path1 == path2) continue
try {
val comparison = comparator.compare(path1, path2)
if (comparison != 0) return@Comparator comparison
} catch (_: Throwable) {
// possibly happens if the version is invalid.
continue
}
}
0
}
}
fun readOrEmpty(path: Path): RuntimeData {
return try {
json.decodeFromString(path.readString())
} catch (e: Throwable) {
when (e) {
is NoSuchFileException,
is FileNotFoundException -> EMPTY
is SerializationException ->
throw DocGeneratorBugException("Error deserializing `${path.toUri()}`.", e)
else -> throw e
}
}
}
}
fun <T : ElementRef<*>> addKnownVersions(
myRef: T,
versions: Set<String>?,
comparator: Comparator<String>,
): Pair<RuntimeData, Boolean> {
if (versions == null) return this to false
val newEffectiveVersions = knownVersions.mapTo(mutableSetOf()) { it.text } + versions
val knownVersions =
newEffectiveVersions
.sortedWith(comparator)
.map { version -> RuntimeDataLink(text = version, href = myRef.pageUrlForVersion(version)) }
.toSet()
if (knownVersions == this.knownVersions) {
return this to false
}
return copy(knownVersions = knownVersions) to true
}
fun <T : ElementRef<*>> addKnownUsages(
myRef: T,
refs: Collection<T>?,
text: (T) -> String,
comparator: Comparator<String>,
): Pair<RuntimeData, Boolean> {
if (refs == null) return this to false
val newLinks =
refs.mapTo(mutableSetOf()) { ref ->
RuntimeDataLink(text = text(ref), href = ref.pageUrlRelativeTo(myRef))
}
val knownUsages =
(this.knownUsages + newLinks).distinctByCommparator(comparator).sortedBy { it.text }.toSet()
if (knownUsages == this.knownUsages) {
return this to false
}
return copy(knownUsages = knownUsages) to true
}
fun addKnownSubtypes(
myRef: TypeRef,
subtypes: Collection<TypeRef>?,
comparator: Comparator<String>,
): Pair<RuntimeData, Boolean> {
if (subtypes == null) return this to false
val newLinks =
subtypes.mapTo(mutableSetOf()) { ref ->
RuntimeDataLink(text = ref.displayName, href = ref.pageUrlRelativeTo(myRef))
}
val knownSubtypes =
(this.knownUsages + newLinks).distinctByCommparator(comparator).sortedBy { it.text }.toSet()
if (knownSubtypes == this.knownSubtypes) {
return this to false
}
return copy(knownSubtypes = knownSubtypes) to true
}
fun Collection<RuntimeDataLink>.distinctByCommparator(
comparator: Comparator<String>
): Collection<RuntimeDataLink> {
val compareBySegment = segmentComparator(comparator)
val highestVersions = mutableMapOf<String, RuntimeDataLink>()
for (link in this) {
val prev = highestVersions[link.text]
if (prev == null || compareBySegment.compare(prev, link) > 0) {
highestVersions[link.text] = link
}
}
return highestVersions.values
}
fun writeTo(path: Path) {
path.createParentDirectories()
path.writeString(json.encodeToString(this))
}
fun perPackage(): RuntimeData = copy(knownUsages = setOf(), knownSubtypes = setOf())
fun perPackageVersion(): RuntimeData = RuntimeData(knownVersions = setOf())
}

View File

@@ -15,37 +15,37 @@
*/
package org.pkl.doc
import java.io.OutputStream
import java.nio.file.Path
import kotlin.io.path.ExperimentalPathApi
import kotlin.io.path.deleteRecursively
import org.pkl.core.util.json.JsonWriter
import kotlin.io.path.isRegularFile
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import kotlinx.serialization.ExperimentalSerializationApi
import org.pkl.commons.lazyWithReceiver
// Note: we don't currently make use of persisted type alias data (needs more thought).
@OptIn(ExperimentalPathApi::class)
@OptIn(ExperimentalPathApi::class, ExperimentalSerializationApi::class)
internal class RuntimeDataGenerator(
private val descendingVersionComparator: Comparator<String>,
private val outputDir: Path,
) {
consoleOut: OutputStream,
) : AbstractGenerator(consoleOut) {
private val packageVersions = mutableMapOf<PackageId, MutableSet<String>>()
private val moduleVersions = mutableMapOf<ModuleId, MutableSet<String>>()
private val classVersions = mutableMapOf<TypeId, MutableSet<String>>()
private val packageUsages = mutableMapOf<PackageRef, MutableSet<PackageRef>>()
private val packageVersions: MutableMap<PackageId, MutableSet<PackageRef>> = mutableMapOf()
private val classVersions: MutableMap<TypeId, MutableSet<String>> = mutableMapOf()
private val packageUsages: MutableMap<PackageRef, MutableSet<PackageRef>> = mutableMapOf()
private val typeUsages = mutableMapOf<TypeRef, MutableSet<TypeRef>>()
private val subtypes = mutableMapOf<TypeRef, MutableSet<TypeRef>>()
private val subtypes: MutableMap<TypeRef, MutableSet<TypeRef>> = mutableMapOf()
fun deleteDataDir() {
outputDir.resolve("data").deleteRecursively()
}
fun generate(packages: List<PackageData>) {
suspend fun generate(packages: List<PackageData>) {
collectData(packages)
writeData(packages)
}
private fun collectData(packages: List<PackageData>) {
for (pkg in packages) {
packageVersions.add(pkg.ref.pkg, pkg.ref.version)
packageVersions.add(pkg.ref.pkg, pkg.ref)
for (dependency in pkg.dependencies) {
if (dependency.isStdlib()) continue
// Every package implicitly depends on the stdlib. Showing this dependency adds unwanted
@@ -53,187 +53,168 @@ internal class RuntimeDataGenerator(
packageUsages.add(dependency.ref, pkg.ref)
}
for (module in pkg.modules) {
moduleVersions.add(module.ref.id, module.ref.version)
if (module.moduleClass != null) {
collectData(module.moduleClass)
}
for (clazz in module.classes) {
for (clazz in module.effectiveClasses) {
collectData(clazz)
}
}
}
}
// only collect type use information when belonging to the same package.
private fun collectData(clazz: ClassData) {
classVersions.add(clazz.ref.id, clazz.ref.version)
for (superclass in clazz.superclasses) {
subtypes.add(superclass, clazz.ref)
if (superclass.isInSamePackageAs(clazz.ref)) {
subtypes.add(superclass, clazz.ref)
}
}
for (type in clazz.usedTypes) {
typeUsages.add(type, clazz.ref)
if (type.isInSamePackageAs(clazz.ref)) {
typeUsages.add(type, clazz.ref)
}
}
}
private fun writeData(packages: List<PackageData>) {
for (pkg in packages) {
writePackageFile(pkg.ref)
for (mod in pkg.modules) {
writeModuleFile(mod.ref)
for (clazz in mod.classes) {
writeClassFile(clazz.ref)
val writtenFiles = mutableSetOf<Path>()
private suspend fun writeData(packages: List<PackageData>) {
coroutineScope {
for (pkg in packages) {
launch { writePackageFilePerVersion(pkg) }
for (module in pkg.modules) {
for (clazz in module.effectiveClasses) {
launch { writeClassFilePerVersion(clazz) }
}
}
}
for (pkg in packages.distinctBy { it.ref.pkg }) {
launch { writePackageFile(pkg) }
for (module in pkg.modules) {
for (clazz in module.effectiveClasses) {
launch { writeClassFile(clazz) }
}
}
}
}
updateKnownUsages(packages)
}
private val ModuleData.effectiveClasses: List<ClassData>
get() =
when (moduleClass) {
null -> classes
else ->
buildList {
add(moduleClass)
addAll(classes)
}
}
/**
* It's possible that a new package uses types/packages from things that are already part of the
* docsite.
*
* If so, update the known usages of those things.
*/
private suspend fun updateKnownUsages(packages: List<PackageData>) = coroutineScope {
val newlyWrittenPackageRefs = packages.mapTo(mutableSetOf()) { it.ref }
val existingPackagesWithUpdatedKnownUsages =
packageUsages.keys.filterNot { newlyWrittenPackageRefs.contains(it) }
for (ref in existingPackagesWithUpdatedKnownUsages) {
launch {
val runtimeDataPath = outputDir.resolve(ref.perPackageVersionRuntimeDataPath)
// we must not have this package in our docsite.
if (!runtimeDataPath.isRegularFile()) return@launch
val usages = packageUsages[ref]
val (data, hasNewData) =
ref.existingPerPackageVersionRuntimeData.addKnownUsages(
ref,
usages,
PackageRef::pkg,
descendingVersionComparator,
)
if (hasNewData) {
data.doWriteTo(outputDir.resolve(ref.perPackageVersionRuntimeDataPath))
}
}
}
}
private fun writePackageFile(ref: PackageRef) {
outputDir
.resolve("data/${ref.pkg.pathEncoded}/${ref.version.pathEncoded}/index.js")
.jsonWriter()
.use { writer ->
writer.isLenient = true
writer.writeLinks(
HtmlConstants.KNOWN_VERSIONS,
packageVersions.getOrDefault(ref.pkg, setOf()).sortedWith(descendingVersionComparator),
{ it },
{ if (it == ref.version) null else ref.copy(version = it).pageUrlRelativeTo(ref) },
{ if (it == ref.version) CssConstants.CURRENT_VERSION else null },
)
writer.writeLinks(
HtmlConstants.KNOWN_USAGES,
packageUsages.getOrDefault(ref, setOf()).packagesWithHighestVersions().sortedBy {
it.pkg
},
PackageRef::pkg,
{ it.pageUrlRelativeTo(ref) },
{ null },
)
}
private val ElementRef<*>.existingPerPackageRuntimeData: RuntimeData by lazyWithReceiver {
val path = outputDir.resolve(perPackageRuntimeDataPath)
RuntimeData.readOrEmpty(path)
}
private fun writeModuleFile(ref: ModuleRef) {
outputDir
.resolve(
"data/${ref.pkg.pathEncoded}/${ref.version.pathEncoded}/${ref.module.pathEncoded}/index.js"
private val ElementRef<*>.existingPerPackageVersionRuntimeData: RuntimeData by lazyWithReceiver {
val path = outputDir.resolve(perPackageVersionRuntimeDataPath)
RuntimeData.readOrEmpty(path)
}
private fun RuntimeData.doWriteTo(path: Path) {
writeTo(path)
consoleOut.write("Wrote file ${path.toUri()}\r")
}
private fun RuntimeData.writePerPackageVersion(ref: ElementRef<*>) {
val path = outputDir.resolve(ref.perPackageVersionRuntimeDataPath)
writtenFiles.add(path)
perPackageVersion().writeTo(path)
consoleOut.write("Wrote file ${path.toUri()}\r")
}
private fun RuntimeData.writePerPackage(ref: ElementRef<*>) {
val path = outputDir.resolve(ref.perPackageRuntimeDataPath)
writtenFiles.add(path)
perPackage().writeTo(path)
consoleOut.write("Wrote file ${path.toUri()}\r")
}
private fun writePackageFile(packageData: PackageData) {
val ref = packageData.ref
val newVersions = packageVersions[packageData.ref.pkg]?.mapTo(mutableSetOf()) { it.version }
val (data, _) =
ref.existingPerPackageRuntimeData.addKnownVersions(
ref,
newVersions,
descendingVersionComparator,
)
.jsonWriter()
.use { writer ->
writer.isLenient = true
writer.writeLinks(
HtmlConstants.KNOWN_VERSIONS,
moduleVersions.getOrDefault(ref.id, setOf()).sortedWith(descendingVersionComparator),
{ it },
{ if (it == ref.version) null else ref.copy(version = it).pageUrlRelativeTo(ref) },
{ if (it == ref.version) CssConstants.CURRENT_VERSION else null },
)
writer.writeLinks(
HtmlConstants.KNOWN_USAGES,
typeUsages.getOrDefault(ref.moduleClassRef, setOf()).typesWithHighestVersions().sortedBy {
it.displayName
},
TypeRef::displayName,
{ it.pageUrlRelativeTo(ref) },
{ null },
)
writer.writeLinks(
HtmlConstants.KNOWN_SUBTYPES,
subtypes.getOrDefault(ref.moduleClassRef, setOf()).typesWithHighestVersions().sortedBy {
it.displayName
},
TypeRef::displayName,
{ it.pageUrlRelativeTo(ref) },
{ null },
)
}
data.writePerPackage(ref)
}
private fun writeClassFile(ref: TypeRef) {
outputDir
.resolve(
"data/${ref.pkg.pathEncoded}/${ref.version.pathEncoded}/${ref.module.pathEncoded}/${ref.type.pathEncoded}.js"
private fun writePackageFilePerVersion(packageData: PackageData) {
val ref = packageData.ref
val (data, _) =
ref.existingPerPackageVersionRuntimeData.addKnownUsages(
ref,
packageUsages[ref],
{ it.pkg },
descendingVersionComparator,
)
.jsonWriter()
.use { writer ->
writer.isLenient = true
writer.writeLinks(
HtmlConstants.KNOWN_VERSIONS,
classVersions.getOrDefault(ref.id, setOf()).sortedWith(descendingVersionComparator),
{ it },
{ if (it == ref.version) null else ref.copy(version = it).pageUrlRelativeTo(ref) },
{ if (it == ref.version) CssConstants.CURRENT_VERSION else null },
)
writer.writeLinks(
HtmlConstants.KNOWN_USAGES,
typeUsages.getOrDefault(ref, setOf()).typesWithHighestVersions().sortedBy {
it.displayName
},
TypeRef::displayName,
{ it.pageUrlRelativeTo(ref) },
{ null },
)
writer.writeLinks(
HtmlConstants.KNOWN_SUBTYPES,
subtypes.getOrDefault(ref, setOf()).typesWithHighestVersions().sortedBy {
it.displayName
},
TypeRef::displayName,
{ it.pageUrlRelativeTo(ref) },
{ null },
)
}
data.writePerPackageVersion(ref)
}
private fun <T> JsonWriter.writeLinks(
// HTML element ID
id: String,
// items based on which links are generated
items: List<T>,
// link text
text: (T) -> String,
// link href
href: (T) -> String?,
// link CSS classes
classes: (T) -> String?,
) {
if (items.isEmpty()) return
rawText("runtimeData.links('")
rawText(id)
rawText("','")
array {
for (item in items) {
obj {
name("text").value(text(item))
name("href").value(href(item))
name("classes").value(classes(item))
}
}
}
rawText("');\n")
private fun writeClassFile(classData: ClassData) {
val ref = classData.ref
val newVersions = classVersions[ref.id]?.mapTo(mutableSetOf()) { it }
val (data, _) =
ref.existingPerPackageRuntimeData.addKnownVersions(
ref,
newVersions,
descendingVersionComparator,
)
data.writePerPackage(ref)
}
private fun Set<PackageRef>.packagesWithHighestVersions(): Collection<PackageRef> {
val highestVersions = mutableMapOf<PackageId, PackageRef>()
for (ref in this) {
val prev = highestVersions[ref.pkg]
if (prev == null || descendingVersionComparator.compare(prev.version, ref.version) > 0) {
highestVersions[ref.pkg] = ref
}
}
return highestVersions.values
}
private fun Set<TypeRef>.typesWithHighestVersions(): Collection<TypeRef> {
val highestVersions = mutableMapOf<TypeId, TypeRef>()
for (ref in this) {
val prev = highestVersions[ref.id]
if (prev == null || descendingVersionComparator.compare(prev.version, ref.version) > 0) {
highestVersions[ref.id] = ref
}
}
return highestVersions.values
private fun writeClassFilePerVersion(classData: ClassData) {
val ref = classData.ref
val newSubtypes = subtypes[ref]
val (data, _) =
ref.existingPerPackageVersionRuntimeData.addKnownSubtypes(
ref,
newSubtypes,
descendingVersionComparator,
)
data.writePerPackageVersion(ref)
}
private fun <K, V> MutableMap<K, MutableSet<V>>.add(key: K, value: V) {

View File

@@ -15,166 +15,298 @@
*/
package org.pkl.doc
import java.io.OutputStream
import java.nio.file.Path
import kotlin.io.path.bufferedWriter
import kotlin.io.path.createParentDirectories
import kotlin.io.path.isRegularFile
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.json.Json
import org.pkl.commons.readString
import org.pkl.commons.writeString
import org.pkl.core.Member
import org.pkl.core.PClass
import org.pkl.core.PClass.Method
import org.pkl.core.PClass.Property
import org.pkl.core.PClassInfo
import org.pkl.core.PType
import org.pkl.core.util.json.JsonWriter
import org.pkl.core.TypeAlias
internal class SearchIndexGenerator(private val outputDir: Path) {
fun generateSiteIndex(packagesData: List<PackageData>) {
val path = outputDir.resolve("search-index.js").createParentDirectories()
path.jsonWriter().use { writer ->
writer.apply {
// provide data as JSON string rather than JS literal (more flexible and secure)
rawText("searchData='")
array {
for (pkg in packagesData) {
val pkgPath = "${pkg.ref.pkg}/current"
obj {
name("name").value(pkg.ref.pkg)
name("kind").value(0)
name("url").value("$pkgPath/index.html")
if (pkg.deprecation != null) {
name("deprecated").value(true)
}
}
@OptIn(ExperimentalSerializationApi::class)
internal class SearchIndexGenerator(private val outputDir: Path, consoleOut: OutputStream) :
AbstractGenerator(consoleOut) {
companion object {
private const val PREFIX = "searchData='"
private const val POSTFIX = "';\n"
for (module in pkg.modules) {
obj {
name("name").value(module.ref.fullName)
name("kind").value(1)
name("url").value("$pkgPath/${module.ref.module}/index.html")
if (module.deprecation != null) {
name("deprecated").value(true)
}
}
}
val json = Json {
prettyPrint = false
explicitNulls = false
}
}
private object KindSerializer : KSerializer<Kind> {
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("Kind", PrimitiveKind.INT)
override fun serialize(encoder: Encoder, value: Kind) {
encoder.encodeInt(value.value)
}
override fun deserialize(decoder: Decoder): Kind {
val intValue = decoder.decodeInt()
return Kind.fromInt(intValue)
}
}
@Serializable(with = KindSerializer::class)
enum class Kind(val value: Int) {
PACKAGE(0),
MODULE(1),
TYPEALIAS(2),
CLASS(3),
METHOD(4),
PROPERTY(5);
companion object {
fun fromInt(value: Int) =
entries.firstOrNull { it.value == value }
?: throw IllegalArgumentException("Unknown Kind value: $value")
}
}
private val searchIndexFile = outputDir.resolve("search-index.js")
@Serializable
data class SearchIndexEntry(
val name: String,
val kind: Kind,
val url: String,
val sig: String? = null,
val parId: Int? = null,
val deprecated: Boolean? = null,
val aka: List<String>? = null,
)
data class PackageIndexEntry(
val packageEntry: SearchIndexEntry,
val moduleEntries: List<SearchIndexEntry>,
)
private fun List<SearchIndexEntry>.writeTo(path: Path) {
val self = this
val text = buildString {
append(PREFIX)
append(json.encodeToString(self))
append(POSTFIX)
}
path.writeString(text)
writeOutput("Wrote file ${path.toUri()}\r")
}
private fun PackageData.toEntry(): SearchIndexEntry {
val pkgPath = "${ref.pkg.pathEncoded}/current"
return SearchIndexEntry(
name = ref.pkg,
kind = Kind.PACKAGE,
url = "$pkgPath/${ref.packageRelativeHtmlPath}",
deprecated = deprecation?.let { true },
)
}
private fun ModuleData.toEntry(basePath: String): SearchIndexEntry {
return SearchIndexEntry(
name = ref.fullName,
kind = Kind.MODULE,
url = "$basePath/${ref.packageRelativeHtmlPath}",
deprecated = deprecation?.let { true },
)
}
private fun DocModule.toEntry(): SearchIndexEntry {
val moduleSchema = schema
return SearchIndexEntry(
name = moduleSchema.moduleName,
kind = Kind.MODULE,
url = "$path/index.html",
)
.withAnnotations(moduleSchema.moduleClass)
}
private fun Property.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
return SearchIndexEntry(
name = simpleName,
kind = Kind.PROPERTY,
url = "$basePath#$simpleName",
sig = renderSignature(this),
parId = parentId,
)
.withAnnotations(this)
}
private fun Method.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
return SearchIndexEntry(
name = simpleName,
kind = Kind.METHOD,
url = "$basePath#${simpleName.pathEncoded}()",
sig = renderSignature(this),
parId = parentId,
)
.withAnnotations(this)
}
private fun PClass.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
return SearchIndexEntry(
name = simpleName,
kind = Kind.CLASS,
url = "$basePath/${simpleName.pathEncoded}.html",
parId = parentId,
)
.withAnnotations(this)
}
private fun TypeAlias.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
return SearchIndexEntry(
name = simpleName,
kind = Kind.TYPEALIAS,
url = "$basePath#${simpleName.pathEncoded}",
parId = parentId,
)
.withAnnotations(this)
}
@Suppress("UNCHECKED_CAST")
private fun SearchIndexEntry.withAnnotations(member: Member?): SearchIndexEntry {
if (member == null) return this
val deprecatedAnnotation = member.annotations.find { it.classInfo == PClassInfo.Deprecated }
val alsoKnownAs = member.annotations.find { it.classInfo == PClassInfo.AlsoKnownAs }
return copy(
deprecated = deprecatedAnnotation?.let { true },
aka = alsoKnownAs?.let { it["names"] as List<String> },
)
}
internal fun getCurrentSearchIndex(): List<PackageIndexEntry> {
if (!searchIndexFile.isRegularFile()) {
return emptyList()
}
val text = searchIndexFile.readString()
if (!(text.startsWith(PREFIX) && text.endsWith(POSTFIX))) {
writeOutputLine(
"[error] Incorrect existing search-index.js; either doesnt start with prefix '$PREFIX', or end with postfix '$POSTFIX'"
)
return emptyList()
}
val jsonStr = text.substring(PREFIX.length, text.length - POSTFIX.length)
val entries = json.decodeFromString<List<SearchIndexEntry>>(jsonStr)
return buildList {
var i = 0
while (i < entries.size) {
val packageEntry = entries[i]
i++
val moduleEntries = buildList {
while (i < entries.size && entries[i].kind == Kind.MODULE) {
add(entries[i])
i++
}
}
rawText("';\n")
add(PackageIndexEntry(packageEntry = packageEntry, moduleEntries = moduleEntries))
}
}
}
fun buildSearchIndex(packages: List<PackageData>): List<PackageIndexEntry> = buildList {
for (pkg in packages) {
val pkgPath = "${pkg.ref.pkg.pathEncoded}/current"
add(
PackageIndexEntry(
packageEntry = pkg.toEntry(),
moduleEntries = pkg.modules.map { it.toEntry(basePath = pkgPath) },
)
)
}
}
/** Reads the current site index, and adds the set of newly generated packages to the index. */
fun generateSiteIndex(currentPackages: List<PackageData>) {
val searchIndex = buildSearchIndex(currentPackages)
searchIndexFile.createParentDirectories()
val entries = buildList {
for (packageIndexEntry in searchIndex) {
add(packageIndexEntry.packageEntry)
for (module in packageIndexEntry.moduleEntries) {
add(module)
}
}
}
entries.writeTo(searchIndexFile)
}
fun generate(docPackage: DocPackage) {
val path =
outputDir
.resolve("${docPackage.name.pathEncoded}/${docPackage.version}/search-index.js")
.createParentDirectories()
JsonWriter(path.bufferedWriter()).use { writer ->
writer.apply {
serializeNulls = false
// provide data as JSON string rather than JS literal (more flexible and secure)
rawText("searchData='")
var nextId = 0
array {
for (docModule in docPackage.docModules) {
if (docModule.isUnlisted) continue
val entries = buildList {
var nextId = 0
for (docModule in docPackage.docModules) {
if (docModule.isUnlisted) continue
val module = docModule.schema
val moduleId = nextId
val module = docModule.schema
val moduleId = nextId
nextId += 1
add(docModule.toEntry())
val moduleBasePath = docModule.path
for ((_, property) in module.moduleClass.properties) {
if (property.isUnlisted) continue
nextId += 1
add(property.toEntry(parentId = moduleId, basePath = "${moduleBasePath}/index.html"))
}
for ((_, method) in module.moduleClass.methods) {
if (method.isUnlisted) continue
nextId += 1
add(method.toEntry(parentId = moduleId, basePath = "${moduleBasePath}/index.html"))
}
for ((_, clazz) in module.classes) {
if (clazz.isUnlisted) continue
val classId = nextId
nextId += 1
add(clazz.toEntry(parentId = moduleId, basePath = moduleBasePath))
val classBasePath = "${docModule.path}/${clazz.simpleName}.html"
for ((_, property) in clazz.properties) {
if (property.isUnlisted) continue
nextId += 1
obj {
name("name").value(module.moduleName)
name("kind").value(1)
name("url").value("${docModule.path}/index.html")
writeAnnotations(module.moduleClass)
}
add(property.toEntry(parentId = classId, basePath = classBasePath))
}
for ((propertyName, property) in module.moduleClass.properties) {
if (property.isUnlisted) continue
for ((_, method) in clazz.methods) {
if (method.isUnlisted) continue
nextId += 1
obj {
name("name").value(propertyName)
name("kind").value(5)
name("url").value("${docModule.path}/index.html#$propertyName")
name("sig").value(renderSignature(property))
name("parId").value(moduleId)
writeAnnotations(property)
}
}
for ((methodName, method) in module.moduleClass.methods) {
if (method.isUnlisted) continue
nextId += 1
obj {
name("name").value(methodName)
name("kind").value(4)
name("url").value("${docModule.path}/index.html#$methodName()")
name("sig").value(renderSignature(method))
name("parId").value(moduleId)
writeAnnotations(method)
}
}
for ((className, clazz) in module.classes) {
if (clazz.isUnlisted) continue
val classId = nextId
nextId += 1
obj {
name("name").value(className)
name("kind").value(3)
name("url").value("${docModule.path}/$className.html")
name("parId").value(moduleId)
writeAnnotations(clazz)
}
for ((propertyName, property) in clazz.properties) {
if (property.isUnlisted) continue
nextId += 1
obj {
name("name").value(propertyName)
name("kind").value(5)
name("url").value("${docModule.path}/$className.html#$propertyName")
name("sig").value(renderSignature(property))
name("parId").value(classId)
writeAnnotations(property)
}
}
for ((methodName, method) in clazz.methods) {
if (method.isUnlisted) continue
nextId += 1
obj {
name("name").value(methodName)
name("kind").value(4)
name("url").value("${docModule.path}/$className.html#$methodName()")
name("sig").value(renderSignature(method))
name("parId").value(classId)
writeAnnotations(method)
}
}
}
for ((typeAliasName, typeAlias) in module.typeAliases) {
if (typeAlias.isUnlisted) continue
nextId += 1
obj {
name("name").value(typeAliasName)
name("kind").value(2)
name("url").value("${docModule.path}/index.html#$typeAliasName")
name("parId").value(moduleId)
writeAnnotations(typeAlias)
}
}
nextId += 1
add(method.toEntry(parentId = classId, basePath = classBasePath))
}
}
rawText("';\n")
for ((_, typeAlias) in module.typeAliases) {
nextId += 1
add(typeAlias.toEntry(parentId = moduleId, basePath = "${moduleBasePath}/index.html"))
}
}
}
entries.writeTo(path)
}
private fun renderSignature(method: Method): String =
@@ -267,24 +399,4 @@ internal class SearchIndexGenerator(private val outputDir: Path) {
else -> throw AssertionError("Unknown PType: $type")
}
}
private fun JsonWriter.writeAnnotations(member: Member?): JsonWriter {
if (member == null) return this
if (member.annotations.any { it.classInfo == PClassInfo.Deprecated }) {
name("deprecated")
value(true)
}
member.annotations
.find { it.classInfo == PClassInfo.AlsoKnownAs }
?.let { alsoKnownAs ->
name("aka")
array {
@Suppress("UNCHECKED_CAST") for (name in alsoKnownAs["names"] as List<String>) value(name)
}
}
return this
}
}

View File

@@ -16,22 +16,20 @@
package org.pkl.doc
import java.io.InputStream
import java.io.OutputStream
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.bufferedWriter
import java.nio.file.StandardCopyOption
import kotlin.io.path.createParentDirectories
import kotlin.io.path.outputStream
import org.pkl.core.*
import org.pkl.core.util.IoUtils
import org.pkl.core.util.json.JsonWriter
import org.pkl.parser.Lexer
// overwrites any existing file
internal fun copyResource(resourceName: String, targetDir: Path) {
val targetFile = targetDir.resolve(resourceName).apply { createParentDirectories() }
getResourceAsStream(resourceName).use { sourceStream ->
targetFile.outputStream().use { targetStream -> sourceStream.copyTo(targetStream) }
}
Files.copy(getResourceAsStream(resourceName), targetFile, StandardCopyOption.REPLACE_EXISTING)
}
internal fun getResourceAsStreamOrNull(resourceName: String): InputStream? =
@@ -92,23 +90,6 @@ internal fun getDocCommentOverflow(docComment: String?): String? {
}
}
internal fun Path.jsonWriter(): JsonWriter {
createParentDirectories()
return JsonWriter(bufferedWriter()).apply { serializeNulls = false }
}
internal inline fun JsonWriter.obj(body: JsonWriter.() -> Unit) {
beginObject()
body()
endObject()
}
internal inline fun JsonWriter.array(body: JsonWriter.() -> Unit) {
beginArray()
body()
endArray()
}
internal fun String.replaceSourceCodePlaceholders(
path: String,
sourceLocation: Member.SourceLocation,
@@ -150,3 +131,11 @@ internal val String.asIdentifier: String
internal val String.pathEncoded
get(): String = IoUtils.encodePath(this)
fun OutputStream.write(str: String) = write(str.toByteArray(Charsets.UTF_8))
fun OutputStream.writeLine(str: String) = write((str + "\n").toByteArray(Charsets.UTF_8))
operator fun <A, B> org.pkl.core.util.Pair<A, B>.component1(): A = first
operator fun <A, B> org.pkl.core.util.Pair<A, B>.component2(): B = second

View File

@@ -689,30 +689,49 @@ function clearSearch() {
updateSearchResults(null);
}
const updateRuntimeDataWith = (buildAnchor) => (fragmentId, entries) => {
if (!entries) return;
const fragment = document.createDocumentFragment();
let first = true;
for (const entry of entries) {
const a = document.createElement("a");
buildAnchor(entry, a);
if (first) {
first = false;
} else {
fragment.append(", ");
}
fragment.append(a);
}
const element = document.getElementById(fragmentId);
element.append(fragment);
element.classList.remove("hidden"); // dd
element.previousElementSibling.classList.remove("hidden"); // dt
}
// Functions called by JS data scripts.
// noinspection JSUnusedGlobalSymbols
const runtimeData = {
links: (id, linksJson) => {
const links = JSON.parse(linksJson);
const fragment = document.createDocumentFragment();
let first = true;
for (const link of links) {
const {text, href, classes} = link
const a = document.createElement("a");
a.textContent = text;
if (href) a.href = href;
if (classes) a.className = classes;
if (first) {
first = false;
} else {
fragment.append(", ");
knownVersions: (versions, myVersion) => {
updateRuntimeDataWith((entry, anchor) => {
const { text, href } = entry;
anchor.textContent = text;
// noinspection JSUnresolvedReference
if (text === myVersion) {
anchor.className = "current-version";
} else if (href) {
anchor.href = href;
}
fragment.append(a);
})("known-versions", versions);
},
knownUsagesOrSubtypes: updateRuntimeDataWith((entry, anchor) => {
const { text, href } = entry;
anchor.textContent = text;
// noinspection JSUnresolvedReference
anchor.textContent = text;
if (href) {
anchor.href = href;
}
const element = document.getElementById(id);
element.append(fragment);
element.classList.remove("hidden"); // dd
element.previousElementSibling.classList.remove("hidden"); // dt
}
}),
}