mirror of
https://github.com/apple/pkl.git
synced 2026-03-29 13:21:58 +02:00
Introduce pkl-doc model version 2 (#1169)
Currently, in order to update a pkl-doc documentation site, almost the entire existing site is read in order to update metadata like known versions, known subtypes, and more. For example, adding a new version of a package requires that the existing runtime data of all existing versions be updated. Eventually, this causes the required storage size to balloon exponentially to the number of versions. This addresses these limitations by: 1. Updating the runtime data structure to move "known versions" metadata to the package level (the same JSON file is used for all versions). 2. Eliminating known subtype and known usage information at a cross-package level. 3. Generating the search index by consuming the previously generated search index. 4. Generating the main page by consuming the search index. Because this changes how runtime data is stored, an existing docsite needs to be migrated. This also introduces a new migration command, `pkl-doc --migrate`, which transforms an older version of the website into a newer version.
This commit is contained in:
28
pkl-doc/src/main/kotlin/org/pkl/doc/AbstractGenerator.kt
Normal file
28
pkl-doc/src/main/kotlin/org/pkl/doc/AbstractGenerator.kt
Normal file
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
|
||||
abstract class AbstractGenerator(protected val consoleOut: OutputStream) {
|
||||
protected fun writeOutputLine(message: String) {
|
||||
consoleOut.writeLine(message)
|
||||
}
|
||||
|
||||
protected fun writeOutput(message: String) {
|
||||
consoleOut.write(message)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import kotlinx.html.*
|
||||
import org.pkl.core.PClass
|
||||
|
||||
@@ -25,7 +26,16 @@ internal class ClassPageGenerator(
|
||||
clazz: PClass,
|
||||
pageScope: ClassScope,
|
||||
isTestMode: Boolean,
|
||||
) : ModuleOrClassPageGenerator<ClassScope>(docsiteInfo, docModule, clazz, pageScope, isTestMode) {
|
||||
consoleOut: OutputStream,
|
||||
) :
|
||||
ModuleOrClassPageGenerator<ClassScope>(
|
||||
docsiteInfo,
|
||||
docModule,
|
||||
clazz,
|
||||
pageScope,
|
||||
isTestMode,
|
||||
consoleOut,
|
||||
) {
|
||||
override val html: HTML.() -> Unit = {
|
||||
renderHtmlHead()
|
||||
|
||||
@@ -56,12 +66,12 @@ internal class ClassPageGenerator(
|
||||
clazz.annotations,
|
||||
isDeclaration = true,
|
||||
mapOf(
|
||||
MemberInfoKey("Known subtypes", runtimeDataClasses) to
|
||||
MemberInfoKey("Known subtypes in package", runtimeDataClasses) to
|
||||
{
|
||||
id = HtmlConstants.KNOWN_SUBTYPES
|
||||
classes = runtimeDataClasses
|
||||
},
|
||||
MemberInfoKey("Known usages", runtimeDataClasses) to
|
||||
MemberInfoKey("Known usages in package", runtimeDataClasses) to
|
||||
{
|
||||
id = HtmlConstants.KNOWN_USAGES
|
||||
classes = runtimeDataClasses
|
||||
|
||||
@@ -15,11 +15,13 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import java.net.URI
|
||||
import java.net.URISyntaxException
|
||||
import java.nio.file.Path
|
||||
import kotlin.Pair
|
||||
import org.pkl.commons.cli.CliBaseOptions.Companion.getProjectFile
|
||||
import org.pkl.commons.cli.CliBugException
|
||||
import org.pkl.commons.cli.CliCommand
|
||||
import org.pkl.commons.cli.CliException
|
||||
import org.pkl.commons.toPath
|
||||
@@ -33,7 +35,11 @@ import org.pkl.core.packages.*
|
||||
*
|
||||
* For the low-level Pkldoc API, see [DocGenerator].
|
||||
*/
|
||||
class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(options.base) {
|
||||
class CliDocGenerator(
|
||||
private val options: CliDocGeneratorOptions,
|
||||
private val consoleOut: OutputStream = System.out,
|
||||
) : CliCommand(options.base) {
|
||||
constructor(options: CliDocGeneratorOptions) : this(options, System.out)
|
||||
|
||||
private val packageResolver =
|
||||
PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir)
|
||||
@@ -60,6 +66,17 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
|
||||
),
|
||||
)
|
||||
|
||||
private val versions = mutableMapOf<String, Version>()
|
||||
|
||||
private val versionComparator =
|
||||
Comparator<String> { v1, v2 ->
|
||||
versions
|
||||
.getOrPut(v1) { Version.parse(v1) }
|
||||
.compareTo(versions.getOrPut(v2) { Version.parse(v2) })
|
||||
}
|
||||
|
||||
private val docMigrator = DocMigrator(options.outputDir, System.out, versionComparator)
|
||||
|
||||
private fun DependencyMetadata.getPackageDependencies(): List<DocPackageInfo.PackageDependency> {
|
||||
return buildList {
|
||||
for ((_, dependency) in dependencies) {
|
||||
@@ -87,14 +104,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
|
||||
}
|
||||
|
||||
private fun PackageUri.toDocPackageInfo(): DocPackageInfo {
|
||||
val metadataAndChecksum =
|
||||
val (metadata, checksum) =
|
||||
try {
|
||||
packageResolver.getDependencyMetadataAndComputeChecksum(this)
|
||||
} catch (e: PackageLoadError) {
|
||||
throw CliException("Failed to package metadata for $this: ${e.message}")
|
||||
}
|
||||
val metadata = metadataAndChecksum.first
|
||||
val checksum = metadataAndChecksum.second
|
||||
return DocPackageInfo(
|
||||
name = "${uri.authority}${uri.path.substringBeforeLast('@')}",
|
||||
moduleNamePrefix = "${metadata.name}.",
|
||||
@@ -130,6 +145,15 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
|
||||
}
|
||||
|
||||
override fun doRun() {
|
||||
if (options.migrate) {
|
||||
docMigrator.run()
|
||||
return
|
||||
}
|
||||
if (!docMigrator.isUpToDate) {
|
||||
throw CliException(
|
||||
"pkldoc website model is too old (found: ${docMigrator.docsiteVersion}, required: ${DocMigrator.CURRENT_VERSION}). Run `pkldoc --migrate` to migrate the website."
|
||||
)
|
||||
}
|
||||
val docsiteInfoModuleUris = mutableListOf<URI>()
|
||||
val packageInfoModuleUris = mutableListOf<URI>()
|
||||
val regularModuleUris = mutableListOf<URI>()
|
||||
@@ -271,8 +295,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
|
||||
options.normalizedOutputDir,
|
||||
options.isTestMode,
|
||||
options.noSymlinks,
|
||||
consoleOut,
|
||||
docMigrator,
|
||||
)
|
||||
.run()
|
||||
} catch (e: DocGeneratorBugException) {
|
||||
throw CliBugException(e)
|
||||
} catch (e: DocGeneratorException) {
|
||||
throw CliException(e.message!!)
|
||||
}
|
||||
|
||||
@@ -47,6 +47,9 @@ constructor(
|
||||
* however, will create a full copy instead.
|
||||
*/
|
||||
val noSymlinks: Boolean = false,
|
||||
|
||||
/** Migrate existing pkldoc */
|
||||
val migrate: Boolean = false,
|
||||
) {
|
||||
/** [outputDir] after undergoing normalization. */
|
||||
val normalizedOutputDir: Path = base.normalizedWorkingDir.resolveSafely(outputDir)
|
||||
|
||||
@@ -16,9 +16,14 @@
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.IOException
|
||||
import java.io.OutputStream
|
||||
import java.net.URI
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.concurrent.Executor
|
||||
import java.util.concurrent.Executors
|
||||
import kotlin.io.path.*
|
||||
import kotlinx.coroutines.*
|
||||
import org.pkl.commons.copyRecursively
|
||||
import org.pkl.core.ModuleSchema
|
||||
import org.pkl.core.PClassInfo
|
||||
@@ -39,11 +44,11 @@ class DocGenerator(
|
||||
*/
|
||||
private val docsiteInfo: DocsiteInfo,
|
||||
|
||||
/** The modules to generate documentation for, grouped by package. */
|
||||
modules: Map<DocPackageInfo, Collection<ModuleSchema>>,
|
||||
/** The packages to generate documentation for. */
|
||||
packages: Map<DocPackageInfo, Collection<ModuleSchema>>,
|
||||
|
||||
/**
|
||||
* A function to resolve imports in [modules] and [docsiteInfo].
|
||||
* A function to resolve imports in [packages] and [docsiteInfo].
|
||||
*
|
||||
* Module `pkl.base` is resolved with this function even if not explicitly imported.
|
||||
*/
|
||||
@@ -73,16 +78,26 @@ class DocGenerator(
|
||||
* however, will create a full copy instead.
|
||||
*/
|
||||
private val noSymlinks: Boolean = false,
|
||||
) {
|
||||
|
||||
/** The output stream to write logs to. */
|
||||
consoleOut: OutputStream,
|
||||
|
||||
/**
|
||||
* The doc migrator that is used to determine the latest docsite version, as well as update the
|
||||
* version file.
|
||||
*/
|
||||
private val docMigrator: DocMigrator = DocMigrator(outputDir, consoleOut, versionComparator),
|
||||
) : AbstractGenerator(consoleOut) {
|
||||
companion object {
|
||||
const val CURRENT_DIRECTORY_NAME = "current"
|
||||
|
||||
internal fun List<PackageData>.current(
|
||||
versionComparator: Comparator<String>
|
||||
internal fun determineCurrentPackages(
|
||||
packages: List<PackageData>,
|
||||
descendingVersionComparator: Comparator<String>,
|
||||
): List<PackageData> {
|
||||
val comparator =
|
||||
compareBy<PackageData> { it.ref.pkg }.thenBy(versionComparator) { it.ref.version }
|
||||
return asSequence()
|
||||
compareBy<PackageData> { it.ref.pkg }.thenBy(descendingVersionComparator) { it.ref.version }
|
||||
return packages
|
||||
// If matching a semver pattern, remove any version that has a prerelease
|
||||
// version (e.g. SNAPSHOT in 1.2.3-SNAPSHOT)
|
||||
.filter { Version.parseOrNull(it.ref.version)?.preRelease == null }
|
||||
@@ -90,50 +105,131 @@ class DocGenerator(
|
||||
.distinctBy { it.ref.pkg }
|
||||
.toList()
|
||||
}
|
||||
|
||||
/**
|
||||
* The default exeuctor when running the doc generator.
|
||||
*
|
||||
* Uses [Executors.newVirtualThreadPerTaskExecutor] if available (JDK 21). Otherwise, uses
|
||||
* [Executors.newFixedThreadPool] with 64 threads, or the number of available processors
|
||||
* available to the JVM (whichever is higher).
|
||||
*/
|
||||
internal val executor: Executor
|
||||
get() {
|
||||
try {
|
||||
val method = Executors::class.java.getMethod("newVirtualThreadPerTaskExecutor")
|
||||
return method.invoke(null) as Executor
|
||||
} catch (e: Throwable) {
|
||||
when (e) {
|
||||
is NoSuchMethodException,
|
||||
is IllegalAccessException ->
|
||||
return Executors.newFixedThreadPool(
|
||||
64.coerceAtLeast(Runtime.getRuntime().availableProcessors())
|
||||
)
|
||||
else -> throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val descendingVersionComparator: Comparator<String> = versionComparator.reversed()
|
||||
|
||||
private val docPackages: List<DocPackage> = modules.map { DocPackage(it.key, it.value.toList()) }
|
||||
private val docPackages: List<DocPackage> =
|
||||
packages.map { DocPackage(it.key, it.value.toList()) }.filter { !it.isUnlisted }
|
||||
|
||||
private fun tryLoadPackageData(entry: SearchIndexGenerator.PackageIndexEntry): PackageData? {
|
||||
var packageDataFile =
|
||||
outputDir.resolve(entry.packageEntry.url).resolveSibling("package-data.json")
|
||||
if (!Files.exists(packageDataFile)) {
|
||||
// search-index.js in Pkl 0.29 and below did not encode path.
|
||||
// If we get a file does not exist, try again by encoding the path.
|
||||
packageDataFile =
|
||||
outputDir.resolve(entry.packageEntry.url.pathEncoded).resolveSibling("package-data.json")
|
||||
if (!Files.exists((packageDataFile))) {
|
||||
writeOutputLine(
|
||||
"[Warn] likely corrupted search index; missing $packageDataFile. This entry will be removed in the newly generated index."
|
||||
)
|
||||
return null
|
||||
}
|
||||
}
|
||||
writeOutput("Loading package data for ${packageDataFile.toUri()}\r")
|
||||
return PackageData.read(packageDataFile)
|
||||
}
|
||||
|
||||
private suspend fun getCurrentPackages(
|
||||
siteSearchIndex: List<SearchIndexGenerator.PackageIndexEntry>
|
||||
): List<PackageData> = coroutineScope {
|
||||
siteSearchIndex.map { async { tryLoadPackageData(it) } }.awaitAll().filterNotNull()
|
||||
}
|
||||
|
||||
/** Runs this documentation generator. */
|
||||
fun run() {
|
||||
try {
|
||||
val htmlGenerator =
|
||||
HtmlGenerator(docsiteInfo, docPackages, importResolver, outputDir, isTestMode)
|
||||
val searchIndexGenerator = SearchIndexGenerator(outputDir)
|
||||
val packageDataGenerator = PackageDataGenerator(outputDir)
|
||||
val runtimeDataGenerator = RuntimeDataGenerator(descendingVersionComparator, outputDir)
|
||||
fun run() =
|
||||
runBlocking(executor.asCoroutineDispatcher()) {
|
||||
try {
|
||||
if (!docMigrator.isUpToDate) {
|
||||
throw DocGeneratorException(
|
||||
"Docsite is not up to date. Expected: ${DocMigrator.CURRENT_VERSION}. Found: ${docMigrator.docsiteVersion}. Use DocMigrator to migrate the site."
|
||||
)
|
||||
}
|
||||
val htmlGenerator =
|
||||
HtmlGenerator(docsiteInfo, docPackages, importResolver, outputDir, isTestMode, consoleOut)
|
||||
val searchIndexGenerator = SearchIndexGenerator(outputDir, consoleOut)
|
||||
val packageDataGenerator = PackageDataGenerator(outputDir, consoleOut)
|
||||
val runtimeDataGenerator =
|
||||
RuntimeDataGenerator(descendingVersionComparator, outputDir, consoleOut)
|
||||
|
||||
for (docPackage in docPackages) {
|
||||
if (docPackage.isUnlisted) continue
|
||||
coroutineScope {
|
||||
for (docPackage in docPackages) {
|
||||
launch {
|
||||
docPackage.deletePackageDir()
|
||||
coroutineScope {
|
||||
launch { htmlGenerator.generate(docPackage) }
|
||||
launch { searchIndexGenerator.generate(docPackage) }
|
||||
launch { packageDataGenerator.generate(docPackage) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
docPackage.deletePackageDir()
|
||||
htmlGenerator.generate(docPackage)
|
||||
searchIndexGenerator.generate(docPackage)
|
||||
packageDataGenerator.generate(docPackage)
|
||||
writeOutputLine("Generated HTML for packages")
|
||||
|
||||
val newlyGeneratedPackages = docPackages.map(::PackageData).sortedBy { it.ref.pkg }
|
||||
val currentSearchIndex = searchIndexGenerator.getCurrentSearchIndex()
|
||||
|
||||
writeOutputLine("Loaded current search index")
|
||||
|
||||
val existingCurrentPackages = getCurrentPackages(currentSearchIndex)
|
||||
|
||||
writeOutputLine("Fetched latest packages")
|
||||
|
||||
val currentPackages =
|
||||
determineCurrentPackages(
|
||||
newlyGeneratedPackages + existingCurrentPackages,
|
||||
descendingVersionComparator,
|
||||
)
|
||||
|
||||
createCurrentDirectories(currentPackages, existingCurrentPackages)
|
||||
searchIndexGenerator.generateSiteIndex(currentPackages)
|
||||
htmlGenerator.generateSite(currentPackages)
|
||||
runtimeDataGenerator.generate(newlyGeneratedPackages)
|
||||
|
||||
writeOutputLine("Wrote package runtime data files")
|
||||
|
||||
docMigrator.updateDocsiteVersion()
|
||||
} catch (e: IOException) {
|
||||
throw DocGeneratorBugException("I/O error generating documentation: $e", e)
|
||||
}
|
||||
|
||||
val packagesData = packageDataGenerator.readAll()
|
||||
val currentPackagesData = packagesData.current(descendingVersionComparator)
|
||||
|
||||
createCurrentDirectories(currentPackagesData)
|
||||
|
||||
htmlGenerator.generateSite(currentPackagesData)
|
||||
searchIndexGenerator.generateSiteIndex(currentPackagesData)
|
||||
runtimeDataGenerator.deleteDataDir()
|
||||
runtimeDataGenerator.generate(packagesData)
|
||||
} catch (e: IOException) {
|
||||
throw DocGeneratorException("I/O error generating documentation.", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun DocPackage.deletePackageDir() {
|
||||
outputDir.resolve(IoUtils.encodePath("$name/$version")).deleteRecursively()
|
||||
}
|
||||
|
||||
private fun createCurrentDirectories(currentPackagesData: List<PackageData>) {
|
||||
for (packageData in currentPackagesData) {
|
||||
private fun createCurrentDirectories(
|
||||
currentPackages: List<PackageData>,
|
||||
existingCurrentPackages: List<PackageData>,
|
||||
) {
|
||||
val packagesToCreate = currentPackages - existingCurrentPackages
|
||||
for (packageData in packagesToCreate) {
|
||||
val basePath = outputDir.resolve(packageData.ref.pkg.pathEncoded)
|
||||
val src = basePath.resolve(packageData.ref.version)
|
||||
val dst = basePath.resolve(CURRENT_DIRECTORY_NAME)
|
||||
@@ -218,7 +314,7 @@ internal class DocModule(
|
||||
get() = schema.moduleName
|
||||
|
||||
val path: String by lazy {
|
||||
name.substring(parent.docPackageInfo.moduleNamePrefix.length).replace('.', '/')
|
||||
name.pathEncoded.substring(parent.docPackageInfo.moduleNamePrefix.length).replace('.', '/')
|
||||
}
|
||||
|
||||
val overview: String?
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
|
||||
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -15,5 +15,8 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
class DocGeneratorException(message: String, cause: Throwable? = null) :
|
||||
open class DocGeneratorException(message: String, cause: Throwable? = null) :
|
||||
RuntimeException(message, cause)
|
||||
|
||||
class DocGeneratorBugException(message: String, cause: Throwable? = null) :
|
||||
DocGeneratorException(message, cause)
|
||||
|
||||
279
pkl-doc/src/main/kotlin/org/pkl/doc/DocMigrator.kt
Normal file
279
pkl-doc/src/main/kotlin/org/pkl/doc/DocMigrator.kt
Normal file
@@ -0,0 +1,279 @@
|
||||
/*
|
||||
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import kotlin.io.path.ExperimentalPathApi
|
||||
import kotlin.io.path.createParentDirectories
|
||||
import kotlin.io.path.deleteIfExists
|
||||
import kotlin.io.path.exists
|
||||
import kotlin.io.path.isRegularFile
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
import kotlin.io.path.name
|
||||
import kotlin.io.path.readLines
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.pkl.commons.lazyWithReceiver
|
||||
import org.pkl.commons.readString
|
||||
import org.pkl.commons.toUri
|
||||
import org.pkl.commons.writeString
|
||||
import org.pkl.core.util.IoUtils
|
||||
import org.pkl.doc.RuntimeData.Companion.EMPTY
|
||||
|
||||
/** Migrates an existing Pkldoc site from version 1 to version 2. */
|
||||
@OptIn(ExperimentalPathApi::class)
|
||||
class DocMigrator(
|
||||
private val outputDir: Path,
|
||||
consoleOut: OutputStream,
|
||||
versionComparator: Comparator<String>,
|
||||
) : AbstractGenerator(consoleOut) {
|
||||
companion object {
|
||||
const val CURRENT_VERSION = 2
|
||||
|
||||
private val json = Json {
|
||||
ignoreUnknownKeys = true
|
||||
explicitNulls = false
|
||||
}
|
||||
|
||||
private const val LEGACY_KNOWN_VERSIONS_PREFIX =
|
||||
"runtimeData.links('${HtmlConstants.KNOWN_VERSIONS}','"
|
||||
private const val LEGACY_KNOWN_USAGES_PREFIX =
|
||||
"runtimeData.links('${HtmlConstants.KNOWN_USAGES}','"
|
||||
private const val LEGACY_KNOWN_SUBTYPES_PREFIX =
|
||||
"runtimeData.links('${HtmlConstants.KNOWN_SUBTYPES}','"
|
||||
private const val LEGACY_SUFFIX = "');"
|
||||
|
||||
internal fun parseLegacyRuntimeData(path: Path, myVersionHref: String): RuntimeData {
|
||||
try {
|
||||
var runtimeData = EMPTY
|
||||
for (line in Files.lines(path)) {
|
||||
runtimeData =
|
||||
when {
|
||||
line.startsWith(LEGACY_KNOWN_VERSIONS_PREFIX) -> {
|
||||
val knownVersions =
|
||||
readLegacyLine(line, LEGACY_KNOWN_VERSIONS_PREFIX, LEGACY_SUFFIX).mapTo(
|
||||
mutableSetOf()
|
||||
) { link ->
|
||||
// fill in missing href
|
||||
if (link.href != null) link else link.copy(href = myVersionHref)
|
||||
}
|
||||
runtimeData.copy(knownVersions = knownVersions)
|
||||
}
|
||||
|
||||
line.startsWith(LEGACY_KNOWN_USAGES_PREFIX) -> {
|
||||
val knownUsages = readLegacyLine(line, LEGACY_KNOWN_USAGES_PREFIX, LEGACY_SUFFIX)
|
||||
runtimeData.copy(knownUsages = knownUsages)
|
||||
}
|
||||
|
||||
line.startsWith(LEGACY_KNOWN_SUBTYPES_PREFIX) -> {
|
||||
val knownSubtypes =
|
||||
readLegacyLine(line, LEGACY_KNOWN_SUBTYPES_PREFIX, LEGACY_SUFFIX)
|
||||
runtimeData.copy(knownSubtypes = knownSubtypes)
|
||||
}
|
||||
|
||||
else -> throw RuntimeException("Unexpected runtimeData line: $line")
|
||||
}
|
||||
}
|
||||
return runtimeData
|
||||
} catch (e: NoSuchFileException) {
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
private fun readLegacyLine(line: String, prefix: String, suffix: String): Set<RuntimeDataLink> {
|
||||
val jsStr = line.substring(prefix.length, line.length - suffix.length)
|
||||
return json.decodeFromString<List<RuntimeDataLink>>(jsStr).toSet()
|
||||
}
|
||||
}
|
||||
|
||||
val isUpToDate by lazy {
|
||||
if (!Files.exists(outputDir.resolve("index.html"))) {
|
||||
// must be the first run
|
||||
return@lazy true
|
||||
}
|
||||
docsiteVersion == CURRENT_VERSION
|
||||
}
|
||||
|
||||
val docsiteVersion by lazy {
|
||||
if (!versionPath.isRegularFile()) 1 else versionPath.readString().trim().toInt()
|
||||
}
|
||||
|
||||
private val versionPath
|
||||
get() = outputDir.resolve(".pkldoc/VERSION")
|
||||
|
||||
private val descendingVersionComparator = versionComparator.reversed()
|
||||
|
||||
fun updateDocsiteVersion() {
|
||||
versionPath.createParentDirectories()
|
||||
versionPath.writeString(CURRENT_VERSION.toString())
|
||||
}
|
||||
|
||||
private suspend fun migratePackage(pkgData: PackageData, isCurrentVersion: Boolean) {
|
||||
coroutineScope {
|
||||
if (!isCurrentVersion) {
|
||||
launch { migrateRuntimeData(pkgData) }
|
||||
}
|
||||
launch { migrateHtml(pkgData, isCurrentVersion) }
|
||||
}
|
||||
}
|
||||
|
||||
fun run() =
|
||||
runBlocking(Dispatchers.IO) {
|
||||
if (isUpToDate) {
|
||||
consoleOut.writeLine(
|
||||
"Generated pkldoc is already version $CURRENT_VERSION; there's nothing to do."
|
||||
)
|
||||
return@runBlocking
|
||||
}
|
||||
val packageDatas = Files.walk(outputDir).filter { it.name == "package-data.json" }.toList()
|
||||
val count = AtomicInteger(1)
|
||||
for (path in packageDatas) {
|
||||
val pkgData = PackageData.read(path)
|
||||
val isCurrentVersion = path.parent.name == "current"
|
||||
migratePackage(pkgData, isCurrentVersion)
|
||||
if (!isCurrentVersion) {
|
||||
deleteLegacyRuntimeData(pkgData)
|
||||
}
|
||||
consoleOut.write(
|
||||
"Migrated ${count.incrementAndGet()} packages (${pkgData.ref.pkg}@${pkgData.ref.version})\r"
|
||||
)
|
||||
consoleOut.flush()
|
||||
}
|
||||
launch { copyResource("scripts/pkldoc.js", outputDir) }
|
||||
updateDocsiteVersion()
|
||||
consoleOut.writeLine("Finished migration, migrated $count packages.")
|
||||
}
|
||||
|
||||
private suspend fun migrateHtml(packageData: PackageData, isCurrentVersion: Boolean) =
|
||||
coroutineScope {
|
||||
for (ref in packageData.refs) {
|
||||
launch { doMigrateHtml(ref, outputDir.resolveHtmlPath(ref, isCurrentVersion)) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun Path.resolveHtmlPath(ref: ElementRef<*>, isCurrentVersion: Boolean): Path {
|
||||
val effectiveRef = if (isCurrentVersion) ref.withVersion("current") else ref
|
||||
return resolve(effectiveRef.htmlPath)
|
||||
}
|
||||
|
||||
private val migratedCurrentPackages = mutableSetOf<String>()
|
||||
|
||||
private fun doMigrateHtml(ref: ElementRef<*>, path: Path) {
|
||||
val newHtml = buildString {
|
||||
val lines = path.readLines()
|
||||
for ((idx, line) in lines.withIndex()) {
|
||||
if (line.contains(ref.legacyVersionedRuntimeDataPath)) continue
|
||||
|
||||
appendLine(line)
|
||||
|
||||
if (line.contains("scripts/pkldoc.js")) {
|
||||
if (lines.getOrNull(idx + 1)?.contains("<script type=\"module\"") == false) {
|
||||
val packageVersionUrl =
|
||||
IoUtils.relativize(ref.perPackageRuntimeDataPath.toUri(), ref.htmlPath.toUri())
|
||||
.toString()
|
||||
val perPackageVersionUrl =
|
||||
IoUtils.relativize(ref.perPackageVersionRuntimeDataPath.toUri(), ref.htmlPath.toUri())
|
||||
.toString()
|
||||
|
||||
appendLine(
|
||||
""" <script type="module">
|
||||
import perPackageData from "$packageVersionUrl" with { type: "json" }
|
||||
import perPackageVersionData from "$perPackageVersionUrl" with { type: "json" }
|
||||
|
||||
runtimeData.knownVersions(perPackageData.knownVersions, ${json.encodeToString(ref.version)});
|
||||
runtimeData.knownUsagesOrSubtypes("known-subtypes", perPackageVersionData.knownSubtypes);
|
||||
runtimeData.knownUsagesOrSubtypes("known-usages", perPackageVersionData.knownUsages);
|
||||
</script>"""
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
path.writeString(newHtml)
|
||||
}
|
||||
|
||||
private fun getLatestPackageData(pkg: String): PackageData {
|
||||
val currentPackage = outputDir.resolve("${pkg.pathEncoded}/current/package-data.json")
|
||||
if (currentPackage.isRegularFile()) {
|
||||
return PackageData.read(currentPackage)
|
||||
}
|
||||
// it's possible that the "current" path doesn't exist if there are only pre-releases.
|
||||
val versions = currentPackage.parent.parent.listDirectoryEntries()
|
||||
val latestVersion = versions.map { it.name }.sortedWith(descendingVersionComparator).first()
|
||||
return PackageData.read(
|
||||
outputDir.resolve("${pkg.pathEncoded}/$latestVersion/package-data.json")
|
||||
)
|
||||
}
|
||||
|
||||
/** Convert legacy style data paths to new style paths */
|
||||
private suspend fun migrateRuntimeData(packageData: PackageData) = coroutineScope {
|
||||
if (!migratedCurrentPackages.contains(packageData.ref.pkg)) {
|
||||
val currentPackageData = getLatestPackageData(packageData.ref.pkg)
|
||||
for (ref in currentPackageData.refs) {
|
||||
launch { doMigratePerPackageData(ref) }
|
||||
}
|
||||
migratedCurrentPackages.add(packageData.ref.pkg)
|
||||
}
|
||||
for (ref in packageData.refs) {
|
||||
launch { doMigratePerPackageVersionData(ref) }
|
||||
}
|
||||
}
|
||||
|
||||
private val PackageData.refs
|
||||
get(): List<ElementRef<*>> {
|
||||
return buildList {
|
||||
add(ref)
|
||||
for (module in modules) {
|
||||
add(module.ref)
|
||||
for (clazz in module.classes) {
|
||||
add(clazz.ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val ElementRef<*>.legacyRuntimeData: RuntimeData? by lazyWithReceiver {
|
||||
val legacyPath = outputDir.resolve(legacyVersionedRuntimeDataPath)
|
||||
when {
|
||||
legacyPath.exists() ->
|
||||
parseLegacyRuntimeData(legacyPath, myVersionHref = pageUrlForVersion(version))
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
private fun doMigratePerPackageVersionData(ref: ElementRef<*>) {
|
||||
val data = ref.legacyRuntimeData ?: return
|
||||
data.perPackageVersion().writeTo(outputDir.resolve(ref.perPackageVersionRuntimeDataPath))
|
||||
}
|
||||
|
||||
private fun doMigratePerPackageData(ref: ElementRef<*>) {
|
||||
val data = ref.legacyRuntimeData ?: return
|
||||
data.perPackage().writeTo(outputDir.resolve(ref.perPackageRuntimeDataPath))
|
||||
}
|
||||
|
||||
private suspend fun deleteLegacyRuntimeData(packageData: PackageData) = coroutineScope {
|
||||
for (ref in packageData.refs) {
|
||||
launch { outputDir.resolve(ref.legacyVersionedRuntimeDataPath).deleteIfExists() }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,7 +117,10 @@ internal sealed class DocScope {
|
||||
/** A scope that corresponds to an entire Pkldoc page. */
|
||||
internal abstract class PageScope : DocScope() {
|
||||
/** The location of the runtime data file for this page. */
|
||||
abstract val dataUrl: URI
|
||||
abstract val perPackageDataUrl: URI
|
||||
|
||||
/** The location of the runtime data JSON file that is per-version for this page. */
|
||||
abstract val perPackageVersionDataUrl: URI
|
||||
}
|
||||
|
||||
// equality is identity
|
||||
@@ -125,15 +128,15 @@ internal class SiteScope(
|
||||
val docPackages: List<DocPackage>,
|
||||
private val overviewImports: Map<String, URI>,
|
||||
private val importResolver: (URI) -> ModuleSchema,
|
||||
outputDir: Path,
|
||||
val outputDir: Path,
|
||||
) : PageScope() {
|
||||
private val pklVersion = Release.current().version().withBuild(null).toString()
|
||||
|
||||
private val pklBaseModule: ModuleSchema by lazy { importResolver("pkl:base".toUri()) }
|
||||
|
||||
val packageScopes: Map<String, PackageScope> by lazy {
|
||||
val packageScopes: Map<DocPackageInfo, PackageScope> by lazy {
|
||||
docPackages.associate { docPackage ->
|
||||
docPackage.name to
|
||||
docPackage.docPackageInfo to
|
||||
PackageScope(
|
||||
docPackage.docPackageInfo,
|
||||
docPackage.docModules.map { it.schema },
|
||||
@@ -144,6 +147,11 @@ internal class SiteScope(
|
||||
}
|
||||
}
|
||||
|
||||
val stdlibPackageScope: PackageScope? by lazy {
|
||||
val pklPackage = docPackages.find { it.name == "pkl" }
|
||||
packageScopes[pklPackage?.docPackageInfo]
|
||||
}
|
||||
|
||||
private val pklBaseScope: ModuleScope by lazy {
|
||||
ModuleScope(pklBaseModule, resolveModuleNameToDocUrl("pkl.base")!!, null)
|
||||
}
|
||||
@@ -155,7 +163,10 @@ internal class SiteScope(
|
||||
IoUtils.ensurePathEndsWithSlash(outputDir.toUri()).resolve("index.html")
|
||||
}
|
||||
|
||||
override val dataUrl: URI
|
||||
override val perPackageDataUrl: URI
|
||||
get() = throw UnsupportedOperationException("perVersionDataUrl")
|
||||
|
||||
override val perPackageVersionDataUrl: URI
|
||||
get() = throw UnsupportedOperationException("perVersionDataUrl")
|
||||
|
||||
fun createEmptyPackageScope(
|
||||
@@ -187,7 +198,7 @@ internal class SiteScope(
|
||||
|
||||
override fun getProperty(name: String): DocScope? = null
|
||||
|
||||
fun getPackage(name: String): PackageScope = packageScopes.getValue(name)
|
||||
fun getPackage(packageInfo: DocPackageInfo): PackageScope = packageScopes.getValue(packageInfo)
|
||||
|
||||
fun resolveImport(uri: URI): ModuleSchema = importResolver(uri)
|
||||
|
||||
@@ -195,7 +206,7 @@ internal class SiteScope(
|
||||
when {
|
||||
name.startsWith("pkl.") -> {
|
||||
val packagePage =
|
||||
packageScopes["pkl"]?.url // link to locally generated stdlib docs if available
|
||||
stdlibPackageScope?.url // link to locally generated stdlib docs if available
|
||||
?: PklInfo.current().packageIndex.getPackagePage("pkl", pklVersion).toUri()
|
||||
packagePage.resolve(name.substring(4) + "/")
|
||||
}
|
||||
@@ -264,8 +275,12 @@ internal class PackageScope(
|
||||
return IoUtils.relativize(myVersion, scope.url)
|
||||
}
|
||||
|
||||
override val dataUrl: URI by lazy {
|
||||
parent.url.resolve("./data/${name.pathEncoded}/$version/index.js")
|
||||
override val perPackageDataUrl: URI by lazy {
|
||||
parent.url.resolve("./data/${name.pathEncoded}/_/index.json")
|
||||
}
|
||||
|
||||
override val perPackageVersionDataUrl: URI by lazy {
|
||||
parent.url.resolve("./data/${name.pathEncoded}/$version/index.json")
|
||||
}
|
||||
|
||||
fun getModule(name: String): ModuleScope = moduleScopes.getValue(name)
|
||||
@@ -330,7 +345,13 @@ internal class ModuleScope(
|
||||
getModulePath(module.moduleName, parent!!.docPackageInfo.moduleNamePrefix).uriEncoded
|
||||
}
|
||||
|
||||
override val dataUrl: URI by lazy { parent!!.dataUrl.resolve("./$path/index.js") }
|
||||
override val perPackageDataUrl: URI by lazy {
|
||||
parent!!.perPackageDataUrl.resolve("./$path/index.json")
|
||||
}
|
||||
|
||||
override val perPackageVersionDataUrl: URI by lazy {
|
||||
parent!!.perPackageVersionDataUrl.resolve("./$path/index.json")
|
||||
}
|
||||
|
||||
override fun getMethod(name: String): MethodScope? =
|
||||
module.moduleClass.allMethods[name]?.let { MethodScope(it, this) }
|
||||
@@ -393,8 +414,14 @@ internal class ClassScope(
|
||||
else parentUrl.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.html")
|
||||
}
|
||||
|
||||
override val dataUrl: URI by lazy {
|
||||
parent!!.dataUrl.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.js")
|
||||
override val perPackageDataUrl: URI by lazy {
|
||||
parent!!.perPackageDataUrl.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.json")
|
||||
}
|
||||
|
||||
override val perPackageVersionDataUrl: URI by lazy {
|
||||
parent!!
|
||||
.perPackageVersionDataUrl
|
||||
.resolve("${clazz.simpleName.pathEncoded.uriEncodedComponent}.json")
|
||||
}
|
||||
|
||||
override fun getMethod(name: String): MethodScope? =
|
||||
|
||||
@@ -15,8 +15,11 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import java.net.URI
|
||||
import java.nio.file.Path
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import org.pkl.core.ModuleSchema
|
||||
|
||||
internal class HtmlGenerator(
|
||||
@@ -25,67 +28,70 @@ internal class HtmlGenerator(
|
||||
importResolver: (URI) -> ModuleSchema,
|
||||
private val outputDir: Path,
|
||||
private val isTestMode: Boolean,
|
||||
) {
|
||||
consoleOut: OutputStream,
|
||||
) : AbstractGenerator(consoleOut) {
|
||||
private val siteScope =
|
||||
SiteScope(docPackages, docsiteInfo.overviewImports, importResolver, outputDir)
|
||||
|
||||
fun generate(docPackage: DocPackage) {
|
||||
val packageScope = siteScope.getPackage(docPackage.name)
|
||||
|
||||
PackagePageGenerator(docsiteInfo, docPackage, packageScope).run()
|
||||
suspend fun generate(docPackage: DocPackage) = coroutineScope {
|
||||
val packageScope = siteScope.getPackage(docPackage.docPackageInfo)
|
||||
launch { PackagePageGenerator(docsiteInfo, docPackage, packageScope, consoleOut).run() }
|
||||
|
||||
for (docModule in docPackage.docModules) {
|
||||
if (docModule.isUnlisted) continue
|
||||
|
||||
val moduleScope = packageScope.getModule(docModule.name)
|
||||
|
||||
ModulePageGenerator(docsiteInfo, docPackage, docModule, moduleScope, isTestMode).run()
|
||||
launch {
|
||||
ModulePageGenerator(docsiteInfo, docPackage, docModule, moduleScope, isTestMode, consoleOut)
|
||||
.run()
|
||||
}
|
||||
|
||||
for ((_, clazz) in docModule.schema.classes) {
|
||||
if (clazz.isUnlisted) continue
|
||||
|
||||
ClassPageGenerator(
|
||||
docsiteInfo,
|
||||
docPackage,
|
||||
docModule,
|
||||
clazz,
|
||||
ClassScope(clazz, moduleScope.url, moduleScope),
|
||||
isTestMode,
|
||||
)
|
||||
.run()
|
||||
launch {
|
||||
ClassPageGenerator(
|
||||
docsiteInfo,
|
||||
docPackage,
|
||||
docModule,
|
||||
clazz,
|
||||
ClassScope(clazz, moduleScope.url, moduleScope),
|
||||
isTestMode,
|
||||
consoleOut,
|
||||
)
|
||||
.run()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun generateSite(packagesData: List<PackageData>) {
|
||||
MainPageGenerator(docsiteInfo, packagesData, siteScope).run()
|
||||
|
||||
generateStaticResources()
|
||||
suspend fun generateSite(packages: List<PackageData>) = coroutineScope {
|
||||
launch { MainPageGenerator(docsiteInfo, packages, siteScope, consoleOut).run() }
|
||||
launch { generateStaticResources() }
|
||||
}
|
||||
|
||||
private fun generateStaticResources() {
|
||||
copyResource("fonts/lato-v14-latin_latin-ext-regular.woff2", outputDir)
|
||||
copyResource("fonts/lato-v14-latin_latin-ext-700.woff2", outputDir)
|
||||
|
||||
copyResource("fonts/open-sans-v15-latin_latin-ext-regular.woff2", outputDir)
|
||||
copyResource("fonts/open-sans-v15-latin_latin-ext-italic.woff2", outputDir)
|
||||
copyResource("fonts/open-sans-v15-latin_latin-ext-700.woff2", outputDir)
|
||||
copyResource("fonts/open-sans-v15-latin_latin-ext-700italic.woff2", outputDir)
|
||||
|
||||
copyResource("fonts/source-code-pro-v7-latin_latin-ext-regular.woff2", outputDir)
|
||||
copyResource("fonts/source-code-pro-v7-latin_latin-ext-700.woff2", outputDir)
|
||||
|
||||
copyResource("fonts/MaterialIcons-Regular.woff2", outputDir)
|
||||
|
||||
copyResource("scripts/pkldoc.js", outputDir)
|
||||
copyResource("scripts/search-worker.js", outputDir)
|
||||
copyResource("scripts/scroll-into-view.min.js", outputDir)
|
||||
|
||||
copyResource("styles/pkldoc.css", outputDir)
|
||||
|
||||
copyResource("images/apple-touch-icon.png", outputDir)
|
||||
copyResource("images/favicon.svg", outputDir)
|
||||
copyResource("images/favicon-16x16.png", outputDir)
|
||||
copyResource("images/favicon-32x32.png", outputDir)
|
||||
private suspend fun generateStaticResources() = coroutineScope {
|
||||
val resources =
|
||||
listOf(
|
||||
"fonts/lato-v14-latin_latin-ext-regular.woff2",
|
||||
"fonts/lato-v14-latin_latin-ext-700.woff2",
|
||||
"fonts/open-sans-v15-latin_latin-ext-regular.woff2",
|
||||
"fonts/open-sans-v15-latin_latin-ext-italic.woff2",
|
||||
"fonts/open-sans-v15-latin_latin-ext-700.woff2",
|
||||
"fonts/open-sans-v15-latin_latin-ext-700italic.woff2",
|
||||
"fonts/source-code-pro-v7-latin_latin-ext-regular.woff2",
|
||||
"fonts/source-code-pro-v7-latin_latin-ext-700.woff2",
|
||||
"fonts/MaterialIcons-Regular.woff2",
|
||||
"scripts/pkldoc.js",
|
||||
"scripts/search-worker.js",
|
||||
"scripts/scroll-into-view.min.js",
|
||||
"styles/pkldoc.css",
|
||||
"images/apple-touch-icon.png",
|
||||
"images/favicon.svg",
|
||||
"images/favicon-16x16.png",
|
||||
"images/favicon-32x32.png",
|
||||
)
|
||||
for (resource in resources) {
|
||||
launch { copyResource(resource, outputDir) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
|
||||
help = "Module paths/uris, or package uris to generate documentation for",
|
||||
)
|
||||
.convert { parseModuleName(it) }
|
||||
.multiple(required = true)
|
||||
.multiple()
|
||||
|
||||
private val outputDir: Path by
|
||||
option(
|
||||
@@ -72,6 +72,14 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
|
||||
private val isTestMode by
|
||||
option(names = arrayOf("--test-mode"), help = "Internal test mode", hidden = true).flag()
|
||||
|
||||
private val migrate: Boolean by
|
||||
option(
|
||||
names = arrayOf("--migrate"),
|
||||
help = "Migrate a pkl-doc site from version 0 to version 1.",
|
||||
)
|
||||
.single()
|
||||
.flag(default = false)
|
||||
|
||||
private val projectOptions by ProjectOptions()
|
||||
|
||||
override val helpString: String = "Generate HTML documentation from Pkl modules and packages."
|
||||
@@ -83,6 +91,7 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
|
||||
outputDir,
|
||||
isTestMode,
|
||||
noSymlinks,
|
||||
migrate,
|
||||
)
|
||||
CliDocGenerator(options).run()
|
||||
}
|
||||
|
||||
@@ -15,13 +15,15 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import kotlinx.html.*
|
||||
|
||||
internal abstract class MainOrPackagePageGenerator<S>(
|
||||
docsiteInfo: DocsiteInfo,
|
||||
pageScope: S,
|
||||
private val siteScope: SiteScope,
|
||||
) : PageGenerator<S>(docsiteInfo, pageScope) where S : PageScope {
|
||||
consoleOut: OutputStream,
|
||||
) : PageGenerator<S>(docsiteInfo, pageScope, consoleOut) where S : PageScope {
|
||||
protected fun UL.renderModuleOrPackage(
|
||||
name: String,
|
||||
moduleOrPackageScope: DocScope,
|
||||
|
||||
@@ -15,13 +15,15 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import kotlinx.html.*
|
||||
|
||||
internal class MainPageGenerator(
|
||||
docsiteInfo: DocsiteInfo,
|
||||
private val packagesData: List<PackageData>,
|
||||
pageScope: SiteScope,
|
||||
) : MainOrPackagePageGenerator<SiteScope>(docsiteInfo, pageScope, pageScope) {
|
||||
consoleOut: OutputStream,
|
||||
) : MainOrPackagePageGenerator<SiteScope>(docsiteInfo, pageScope, pageScope, consoleOut) {
|
||||
override val html: HTML.() -> Unit = {
|
||||
renderHtmlHead()
|
||||
|
||||
@@ -89,14 +91,12 @@ internal class MainPageGenerator(
|
||||
ul {
|
||||
for (pkg in sortedPackages) {
|
||||
val packageScope =
|
||||
pageScope.packageScopes[pkg.ref.pkg]
|
||||
// create scope for previously generated package
|
||||
?: pageScope.createEmptyPackageScope(
|
||||
pkg.ref.pkg,
|
||||
pkg.ref.version,
|
||||
pkg.sourceCodeUrlScheme,
|
||||
pkg.sourceCode,
|
||||
)
|
||||
pageScope.createEmptyPackageScope(
|
||||
pkg.ref.pkg,
|
||||
pkg.ref.version,
|
||||
pkg.sourceCodeUrlScheme,
|
||||
pkg.sourceCode,
|
||||
)
|
||||
|
||||
val memberDocs =
|
||||
MemberDocs(
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import java.io.StringWriter
|
||||
import kotlinx.html.*
|
||||
import org.pkl.core.Member
|
||||
@@ -31,7 +32,8 @@ internal abstract class ModuleOrClassPageGenerator<S>(
|
||||
protected val clazz: PClass,
|
||||
scope: S,
|
||||
private val isTestMode: Boolean,
|
||||
) : PageGenerator<S>(docsiteInfo, scope) where S : PageScope {
|
||||
consoleOut: OutputStream,
|
||||
) : PageGenerator<S>(docsiteInfo, scope, consoleOut) where S : PageScope {
|
||||
protected fun HtmlBlockTag.renderProperties() {
|
||||
if (!clazz.hasListedProperty) return
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import kotlinx.html.*
|
||||
|
||||
internal class ModulePageGenerator(
|
||||
@@ -23,6 +24,7 @@ internal class ModulePageGenerator(
|
||||
docModule: DocModule,
|
||||
pageScope: ModuleScope,
|
||||
isTestMode: Boolean,
|
||||
consoleOut: OutputStream,
|
||||
) :
|
||||
ModuleOrClassPageGenerator<ModuleScope>(
|
||||
docsiteInfo,
|
||||
@@ -30,6 +32,7 @@ internal class ModulePageGenerator(
|
||||
docModule.schema.moduleClass,
|
||||
pageScope,
|
||||
isTestMode,
|
||||
consoleOut,
|
||||
) {
|
||||
private val module = docModule.schema
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.IOException
|
||||
import java.io.OutputStream
|
||||
import java.net.URI
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.createParentDirectories
|
||||
@@ -24,8 +25,8 @@ import kotlinx.serialization.*
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.pkl.commons.readString
|
||||
import org.pkl.commons.toUri
|
||||
import org.pkl.commons.walk
|
||||
import org.pkl.core.*
|
||||
import org.pkl.core.packages.PackageUri
|
||||
import org.pkl.core.util.IoUtils
|
||||
|
||||
/**
|
||||
@@ -33,7 +34,8 @@ import org.pkl.core.util.IoUtils
|
||||
* previously generated docs in a newly generated doc website. This is useful if there's a problem
|
||||
* with fetching or evaluating the latest package version.
|
||||
*/
|
||||
internal class PackageDataGenerator(private val outputDir: Path) {
|
||||
internal class PackageDataGenerator(private val outputDir: Path, consoleOut: OutputStream) :
|
||||
AbstractGenerator(consoleOut) {
|
||||
fun generate(pkg: DocPackage) {
|
||||
val path =
|
||||
outputDir
|
||||
@@ -43,19 +45,10 @@ internal class PackageDataGenerator(private val outputDir: Path) {
|
||||
.apply { createParentDirectories() }
|
||||
PackageData(pkg).write(path)
|
||||
}
|
||||
|
||||
fun readAll(): List<PackageData> {
|
||||
return outputDir.walk().use { paths ->
|
||||
paths
|
||||
.filter { it.fileName?.toString() == "package-data.json" }
|
||||
.map { PackageData.read(it) }
|
||||
.toList()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Uniquely identifies a specific version of a package, module, class, or type alias. */
|
||||
internal sealed class ElementRef {
|
||||
internal sealed class ElementRef<T : ElementRef<T>> {
|
||||
/** The package name. */
|
||||
abstract val pkg: String
|
||||
|
||||
@@ -66,15 +59,51 @@ internal sealed class ElementRef {
|
||||
abstract val version: String
|
||||
|
||||
/** The Pkldoc page URL of the element relative to its Pkldoc website root. */
|
||||
abstract val pageUrl: URI
|
||||
val htmlPath: String by lazy { "$basePath/${version.pathEncoded}/$packageRelativeHtmlPath" }
|
||||
|
||||
/** The Pkldoc runtime data JSON path for this ref. */
|
||||
val perPackageRuntimeDataPath: String by lazy { "data/$basePath/_/$packageRelativeDataPath" }
|
||||
|
||||
/** The Pkldoc runtime data JSON path for this ref at a per-version level. */
|
||||
val perPackageVersionRuntimeDataPath: String by lazy {
|
||||
"data/$basePath/${version.pathEncoded}/$packageRelativeDataPath"
|
||||
}
|
||||
|
||||
/** The legacy runtime data path () */
|
||||
val legacyVersionedRuntimeDataPath: String by lazy {
|
||||
"data/$basePath/${version.pathEncoded}/${packageRelativeDataPath.substringBeforeLast(".json") + ".js"}"
|
||||
}
|
||||
|
||||
/**
|
||||
* The Pkldoc page URL of the element relative to [other]'s page URL. Assumes that both elements
|
||||
* have the same Pkldoc website root.
|
||||
*/
|
||||
fun pageUrlRelativeTo(other: ElementRef): String {
|
||||
return IoUtils.relativize(pageUrl, other.pageUrl).toString()
|
||||
fun pageUrlRelativeTo(other: ElementRef<*>): String {
|
||||
return IoUtils.relativize(htmlPath.toUri(), other.htmlPath.toUri()).toString()
|
||||
}
|
||||
|
||||
/** The Pkldoc page URL for my element with a different veresion. */
|
||||
fun pageUrlForVersion(version: String): String {
|
||||
val pathToBasePath = IoUtils.relativize(basePath.toUri(), htmlPath.toUri()).toString()
|
||||
return "${pathToBasePath}${version.pathEncoded}/${withVersion(version).packageRelativeHtmlPath}"
|
||||
}
|
||||
|
||||
abstract fun withVersion(version: String): T
|
||||
|
||||
/** The HTML path within the package folder */
|
||||
abstract val packageRelativeHtmlPath: String
|
||||
|
||||
/** The JSON path within the package folder */
|
||||
abstract val packageRelativeDataPath: String
|
||||
|
||||
fun isInSamePackageAs(other: ElementRef<T>): Boolean =
|
||||
pkg == other.pkg && pkgUri == other.pkgUri && version == other.version
|
||||
|
||||
val basePath: String
|
||||
get() =
|
||||
if (pkgUri?.scheme == "package")
|
||||
"${pkgUri!!.authority}${PackageUri(pkgUri!!).pathWithoutVersion}".pathEncoded
|
||||
else pkg.pathEncoded
|
||||
}
|
||||
|
||||
/** Uniquely identifies a specific version of a package. */
|
||||
@@ -88,8 +117,12 @@ internal data class PackageRef(
|
||||
|
||||
/** The package version. */
|
||||
override val version: String,
|
||||
) : ElementRef() {
|
||||
override val pageUrl: URI by lazy { "$pkg/$version/index.html".toUri() }
|
||||
) : ElementRef<PackageRef>() {
|
||||
override val packageRelativeHtmlPath: String = "index.html"
|
||||
|
||||
override val packageRelativeDataPath: String = "index.json"
|
||||
|
||||
override fun withVersion(version: String): PackageRef = copy(version = version)
|
||||
}
|
||||
|
||||
/** Uniquely identifies a specific version of a module. */
|
||||
@@ -106,12 +139,13 @@ internal data class ModuleRef(
|
||||
|
||||
/** The module path. */
|
||||
val module: String,
|
||||
) : ElementRef() {
|
||||
override val pageUrl: URI by lazy { "$pkg/$version/$module/index.html".toUri() }
|
||||
) : ElementRef<ModuleRef>() {
|
||||
override val packageRelativeHtmlPath: String by lazy { "${module.pathEncoded}/index.html" }
|
||||
|
||||
val moduleClassRef: TypeRef by lazy {
|
||||
TypeRef(pkg, pkgUri, version, module, PClassInfo.MODULE_CLASS_NAME)
|
||||
}
|
||||
override val packageRelativeDataPath: String
|
||||
get() = "$module/index.json".pathEncoded
|
||||
|
||||
override fun withVersion(version: String): ModuleRef = copy(version = version)
|
||||
|
||||
val id: ModuleId by lazy { ModuleId(pkg, module) }
|
||||
|
||||
@@ -138,20 +172,32 @@ internal data class TypeRef(
|
||||
|
||||
/** Whether this is a type alias rather than a class. */
|
||||
val isTypeAlias: Boolean = false,
|
||||
) : ElementRef() {
|
||||
) : ElementRef<TypeRef>() {
|
||||
|
||||
val id: TypeId by lazy { TypeId(pkg, module, type) }
|
||||
|
||||
val displayName: String by lazy { if (isModuleClass) module.substringAfterLast('/') else type }
|
||||
|
||||
override val pageUrl: URI by lazy {
|
||||
override val packageRelativeHtmlPath: String by lazy {
|
||||
when {
|
||||
isTypeAlias -> "$pkg/$version/$module/index.html#$type".toUri()
|
||||
isModuleClass -> "$pkg/$version/$module/index.html".toUri()
|
||||
else -> "$pkg/$version/$module/$type.html".toUri()
|
||||
isTypeAlias -> "$module/index.html#$type".pathEncoded
|
||||
isModuleClass -> "$module/index.html".pathEncoded
|
||||
else -> "$module/$type.html".pathEncoded
|
||||
}
|
||||
}
|
||||
|
||||
override val packageRelativeDataPath: String
|
||||
get() {
|
||||
// typealiases don't have their own runtime data.
|
||||
require(!isTypeAlias) { "typealiases don't have runtime data" }
|
||||
return when {
|
||||
isModuleClass -> "$module/index.json".pathEncoded
|
||||
else -> "$module/$type.json".pathEncoded
|
||||
}
|
||||
}
|
||||
|
||||
override fun withVersion(version: String): TypeRef = copy(version = version)
|
||||
|
||||
private val isModuleClass: Boolean
|
||||
get() = type == PClassInfo.MODULE_CLASS_NAME
|
||||
}
|
||||
@@ -200,13 +246,13 @@ internal class PackageData(
|
||||
try {
|
||||
path.readString()
|
||||
} catch (e: IOException) {
|
||||
throw DocGeneratorException("I/O error reading `$path`.", e)
|
||||
throw DocGeneratorBugException("I/O error reading `${path.toUri()}`.", e)
|
||||
}
|
||||
|
||||
return try {
|
||||
json.decodeFromString(jsonStr)
|
||||
} catch (e: SerializationException) {
|
||||
throw DocGeneratorException("Error deserializing `$path`.", e)
|
||||
throw DocGeneratorBugException("Error deserializing `${path.toUri()}`.", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -228,14 +274,14 @@ internal class PackageData(
|
||||
try {
|
||||
json.encodeToString(this)
|
||||
} catch (e: SerializationException) {
|
||||
throw DocGeneratorException("Error serializing `$path`.", e)
|
||||
throw DocGeneratorBugException("Error serializing `$path`.", e)
|
||||
}
|
||||
|
||||
try {
|
||||
path.createParentDirectories()
|
||||
path.writer().use { it.write(jsonStr) }
|
||||
} catch (e: IOException) {
|
||||
throw DocGeneratorException("I/O error writing `$path`.", e)
|
||||
throw DocGeneratorBugException("I/O error writing `$path`.", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,13 +15,15 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import kotlinx.html.*
|
||||
|
||||
internal class PackagePageGenerator(
|
||||
docsiteInfo: DocsiteInfo,
|
||||
private val docPackage: DocPackage,
|
||||
pageScope: PackageScope,
|
||||
) : MainOrPackagePageGenerator<PackageScope>(docsiteInfo, pageScope, pageScope.parent) {
|
||||
consoleOut: OutputStream,
|
||||
) : MainOrPackagePageGenerator<PackageScope>(docsiteInfo, pageScope, pageScope.parent, consoleOut) {
|
||||
override val html: HTML.() -> Unit = {
|
||||
renderHtmlHead()
|
||||
|
||||
|
||||
@@ -15,10 +15,13 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import kotlin.io.path.bufferedWriter
|
||||
import kotlin.io.path.createParentDirectories
|
||||
import kotlin.io.path.exists
|
||||
import kotlinx.html.*
|
||||
import kotlinx.html.stream.appendHTML
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.commonmark.ext.gfm.tables.TablesExtension
|
||||
import org.commonmark.parser.Parser
|
||||
import org.commonmark.renderer.html.HtmlRenderer
|
||||
@@ -29,7 +32,12 @@ import org.pkl.core.util.IoUtils
|
||||
internal abstract class PageGenerator<out S>(
|
||||
protected val docsiteInfo: DocsiteInfo,
|
||||
protected val pageScope: S,
|
||||
) where S : PageScope {
|
||||
consoleOut: OutputStream,
|
||||
) : AbstractGenerator(consoleOut) where S : PageScope {
|
||||
companion object {
|
||||
private val json = Json {}
|
||||
}
|
||||
|
||||
private val markdownInlineParserFactory = MarkdownParserFactory(pageScope)
|
||||
|
||||
private val markdownParser =
|
||||
@@ -47,33 +55,62 @@ internal abstract class PageGenerator<out S>(
|
||||
fun run() {
|
||||
val path = pageScope.url.toPath()
|
||||
path.createParentDirectories()
|
||||
path.bufferedWriter().use {
|
||||
it.appendLine("<!DOCTYPE html>")
|
||||
it.appendHTML().html(null, html)
|
||||
path.bufferedWriter().use { writer ->
|
||||
writer.appendLine("<!DOCTYPE html>")
|
||||
writer.appendHTML().html(null, html)
|
||||
}
|
||||
writeOutput("Wrote file ${pageScope.url}\r")
|
||||
}
|
||||
|
||||
protected abstract val html: HTML.() -> Unit
|
||||
|
||||
protected abstract fun HTMLTag.renderPageTitle()
|
||||
|
||||
private fun DocScope?.getVersion(): String? {
|
||||
return when (this) {
|
||||
null -> null
|
||||
is SiteScope -> null
|
||||
is PackageScope -> version
|
||||
else -> parent?.getVersion()
|
||||
}
|
||||
}
|
||||
|
||||
protected fun HTML.renderHtmlHead() {
|
||||
lang = "en-US"
|
||||
|
||||
head {
|
||||
meta { charset = "UTF-8" }
|
||||
title { renderPageTitle() }
|
||||
script {
|
||||
src = pageScope.relativeSiteUrl.resolve("scripts/pkldoc.js").toString()
|
||||
defer = true
|
||||
}
|
||||
pageScope.getVersion()?.let { version ->
|
||||
script {
|
||||
type = "module"
|
||||
unsafe {
|
||||
val packageVersionUrl =
|
||||
IoUtils.relativize(pageScope.perPackageDataUrl, pageScope.url).toString()
|
||||
val perPackageVersionUrl =
|
||||
IoUtils.relativize(pageScope.perPackageVersionDataUrl, pageScope.url).toString()
|
||||
|
||||
raw(
|
||||
"""
|
||||
import perPackageData from "$packageVersionUrl" with { type: "json" }
|
||||
import perPackageVersionData from "$perPackageVersionUrl" with { type: "json" }
|
||||
|
||||
runtimeData.knownVersions(perPackageData.knownVersions, ${json.encodeToString(version)});
|
||||
runtimeData.knownUsagesOrSubtypes("known-subtypes", perPackageVersionData.knownSubtypes);
|
||||
runtimeData.knownUsagesOrSubtypes("known-usages", perPackageVersionData.knownUsages);
|
||||
"""
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
script {
|
||||
src = pageScope.relativeSiteUrl.resolve("scripts/scroll-into-view.min.js").toString()
|
||||
defer = true
|
||||
}
|
||||
if (pageScope !is SiteScope) {
|
||||
script {
|
||||
src = IoUtils.relativize(pageScope.dataUrl, pageScope.url).toString()
|
||||
defer = true
|
||||
}
|
||||
}
|
||||
link {
|
||||
href = pageScope.relativeSiteUrl.resolve("styles/pkldoc.css").toString()
|
||||
media = "screen"
|
||||
@@ -102,7 +139,6 @@ internal abstract class PageGenerator<out S>(
|
||||
sizes = "16x16"
|
||||
href = pageScope.relativeSiteUrl.resolve("images/favicon-16x16.png").toString()
|
||||
}
|
||||
meta { charset = "UTF-8" }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,10 +163,13 @@ internal abstract class PageGenerator<out S>(
|
||||
div {
|
||||
id = "search"
|
||||
|
||||
i {
|
||||
id = "search-icon"
|
||||
classes = setOf("material-icons")
|
||||
+"search"
|
||||
label {
|
||||
htmlFor = "search-input"
|
||||
i {
|
||||
id = "search-icon"
|
||||
classes = setOf("material-icons")
|
||||
+"search"
|
||||
}
|
||||
}
|
||||
|
||||
input {
|
||||
@@ -562,7 +601,25 @@ internal abstract class PageGenerator<out S>(
|
||||
return siteScope.packageScopes.values
|
||||
.find { it.name == dep.qualifiedName }
|
||||
?.urlForVersionRelativeTo(pageScope, dep.version)
|
||||
?.toString()
|
||||
?.toString() ?: findAlreadyExistingPackageScope(siteScope, dep.qualifiedName, dep.version)
|
||||
}
|
||||
|
||||
private fun findAlreadyExistingPackageScope(
|
||||
siteScope: SiteScope,
|
||||
name: String,
|
||||
version: String,
|
||||
): String? {
|
||||
// hack: determine if we know about this dependency just by checking to see if the directory
|
||||
// exists.
|
||||
// we don't need to know if the specific _version_ exists (we assume that the version must be
|
||||
// published by the docsite and tolerate possibly broken links).
|
||||
val pkgPath = siteScope.outputDir.resolve(name.pathEncoded)
|
||||
if (pkgPath.exists()) {
|
||||
val targetPath = pkgPath.resolve("$version/index.html")
|
||||
val myUrl = pageScope.url
|
||||
return IoUtils.relativize(targetPath.toUri(), myUrl).toString()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
protected class MemberInfoKey(val name: String, val classes: Set<String> = setOf())
|
||||
@@ -619,12 +676,12 @@ internal abstract class PageGenerator<out S>(
|
||||
}
|
||||
}
|
||||
|
||||
result[MemberInfoKey("Known subtypes", runtimeDataClasses)] = {
|
||||
result[MemberInfoKey("Known subtypes in package", runtimeDataClasses)] = {
|
||||
id = HtmlConstants.KNOWN_SUBTYPES
|
||||
classes = runtimeDataClasses
|
||||
}
|
||||
|
||||
result[MemberInfoKey("Known usages", runtimeDataClasses)] = {
|
||||
result[MemberInfoKey("Known usages in package", runtimeDataClasses)] = {
|
||||
id = HtmlConstants.KNOWN_USAGES
|
||||
classes = runtimeDataClasses
|
||||
}
|
||||
|
||||
160
pkl-doc/src/main/kotlin/org/pkl/doc/RuntimeData.kt
Normal file
160
pkl-doc/src/main/kotlin/org/pkl/doc/RuntimeData.kt
Normal file
@@ -0,0 +1,160 @@
|
||||
/*
|
||||
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.FileNotFoundException
|
||||
import java.nio.file.NoSuchFileException
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.createParentDirectories
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.SerializationException
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.pkl.commons.readString
|
||||
import org.pkl.commons.writeString
|
||||
|
||||
@Serializable internal data class RuntimeDataLink(val text: String, val href: String?)
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
@Serializable
|
||||
internal data class RuntimeData(
|
||||
val knownVersions: Set<RuntimeDataLink> = setOf(),
|
||||
val knownUsages: Set<RuntimeDataLink> = setOf(),
|
||||
val knownSubtypes: Set<RuntimeDataLink> = setOf(),
|
||||
) {
|
||||
|
||||
companion object {
|
||||
val EMPTY = RuntimeData()
|
||||
|
||||
private val json = Json {
|
||||
prettyPrint = true
|
||||
explicitNulls = false
|
||||
ignoreUnknownKeys = true
|
||||
prettyPrintIndent = " "
|
||||
}
|
||||
|
||||
/** Compare two paths, comparing each segment. */
|
||||
private fun segmentComparator(comparator: Comparator<String>): Comparator<RuntimeDataLink> {
|
||||
return Comparator { o1, o2 ->
|
||||
val path1Segments = o1.href!!.split("/")
|
||||
val path2Segments = o2.href!!.split("/")
|
||||
for ((path1, path2) in path1Segments.zip(path2Segments)) {
|
||||
if (path1 == path2) continue
|
||||
try {
|
||||
val comparison = comparator.compare(path1, path2)
|
||||
if (comparison != 0) return@Comparator comparison
|
||||
} catch (_: Throwable) {
|
||||
// possibly happens if the version is invalid.
|
||||
continue
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
fun readOrEmpty(path: Path): RuntimeData {
|
||||
return try {
|
||||
json.decodeFromString(path.readString())
|
||||
} catch (e: Throwable) {
|
||||
when (e) {
|
||||
is NoSuchFileException,
|
||||
is FileNotFoundException -> EMPTY
|
||||
is SerializationException ->
|
||||
throw DocGeneratorBugException("Error deserializing `${path.toUri()}`.", e)
|
||||
else -> throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun <T : ElementRef<*>> addKnownVersions(
|
||||
myRef: T,
|
||||
versions: Set<String>?,
|
||||
comparator: Comparator<String>,
|
||||
): Pair<RuntimeData, Boolean> {
|
||||
if (versions == null) return this to false
|
||||
val newEffectiveVersions = knownVersions.mapTo(mutableSetOf()) { it.text } + versions
|
||||
val knownVersions =
|
||||
newEffectiveVersions
|
||||
.sortedWith(comparator)
|
||||
.map { version -> RuntimeDataLink(text = version, href = myRef.pageUrlForVersion(version)) }
|
||||
.toSet()
|
||||
if (knownVersions == this.knownVersions) {
|
||||
return this to false
|
||||
}
|
||||
return copy(knownVersions = knownVersions) to true
|
||||
}
|
||||
|
||||
fun <T : ElementRef<*>> addKnownUsages(
|
||||
myRef: T,
|
||||
refs: Collection<T>?,
|
||||
text: (T) -> String,
|
||||
comparator: Comparator<String>,
|
||||
): Pair<RuntimeData, Boolean> {
|
||||
if (refs == null) return this to false
|
||||
val newLinks =
|
||||
refs.mapTo(mutableSetOf()) { ref ->
|
||||
RuntimeDataLink(text = text(ref), href = ref.pageUrlRelativeTo(myRef))
|
||||
}
|
||||
val knownUsages =
|
||||
(this.knownUsages + newLinks).distinctByCommparator(comparator).sortedBy { it.text }.toSet()
|
||||
if (knownUsages == this.knownUsages) {
|
||||
return this to false
|
||||
}
|
||||
return copy(knownUsages = knownUsages) to true
|
||||
}
|
||||
|
||||
fun addKnownSubtypes(
|
||||
myRef: TypeRef,
|
||||
subtypes: Collection<TypeRef>?,
|
||||
comparator: Comparator<String>,
|
||||
): Pair<RuntimeData, Boolean> {
|
||||
if (subtypes == null) return this to false
|
||||
val newLinks =
|
||||
subtypes.mapTo(mutableSetOf()) { ref ->
|
||||
RuntimeDataLink(text = ref.displayName, href = ref.pageUrlRelativeTo(myRef))
|
||||
}
|
||||
val knownSubtypes =
|
||||
(this.knownUsages + newLinks).distinctByCommparator(comparator).sortedBy { it.text }.toSet()
|
||||
if (knownSubtypes == this.knownSubtypes) {
|
||||
return this to false
|
||||
}
|
||||
return copy(knownSubtypes = knownSubtypes) to true
|
||||
}
|
||||
|
||||
fun Collection<RuntimeDataLink>.distinctByCommparator(
|
||||
comparator: Comparator<String>
|
||||
): Collection<RuntimeDataLink> {
|
||||
val compareBySegment = segmentComparator(comparator)
|
||||
val highestVersions = mutableMapOf<String, RuntimeDataLink>()
|
||||
for (link in this) {
|
||||
val prev = highestVersions[link.text]
|
||||
if (prev == null || compareBySegment.compare(prev, link) > 0) {
|
||||
highestVersions[link.text] = link
|
||||
}
|
||||
}
|
||||
return highestVersions.values
|
||||
}
|
||||
|
||||
fun writeTo(path: Path) {
|
||||
path.createParentDirectories()
|
||||
path.writeString(json.encodeToString(this))
|
||||
}
|
||||
|
||||
fun perPackage(): RuntimeData = copy(knownUsages = setOf(), knownSubtypes = setOf())
|
||||
|
||||
fun perPackageVersion(): RuntimeData = RuntimeData(knownVersions = setOf())
|
||||
}
|
||||
@@ -15,37 +15,37 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.ExperimentalPathApi
|
||||
import kotlin.io.path.deleteRecursively
|
||||
import org.pkl.core.util.json.JsonWriter
|
||||
import kotlin.io.path.isRegularFile
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import org.pkl.commons.lazyWithReceiver
|
||||
|
||||
// Note: we don't currently make use of persisted type alias data (needs more thought).
|
||||
@OptIn(ExperimentalPathApi::class)
|
||||
@OptIn(ExperimentalPathApi::class, ExperimentalSerializationApi::class)
|
||||
internal class RuntimeDataGenerator(
|
||||
private val descendingVersionComparator: Comparator<String>,
|
||||
private val outputDir: Path,
|
||||
) {
|
||||
consoleOut: OutputStream,
|
||||
) : AbstractGenerator(consoleOut) {
|
||||
|
||||
private val packageVersions = mutableMapOf<PackageId, MutableSet<String>>()
|
||||
private val moduleVersions = mutableMapOf<ModuleId, MutableSet<String>>()
|
||||
private val classVersions = mutableMapOf<TypeId, MutableSet<String>>()
|
||||
private val packageUsages = mutableMapOf<PackageRef, MutableSet<PackageRef>>()
|
||||
private val packageVersions: MutableMap<PackageId, MutableSet<PackageRef>> = mutableMapOf()
|
||||
private val classVersions: MutableMap<TypeId, MutableSet<String>> = mutableMapOf()
|
||||
private val packageUsages: MutableMap<PackageRef, MutableSet<PackageRef>> = mutableMapOf()
|
||||
private val typeUsages = mutableMapOf<TypeRef, MutableSet<TypeRef>>()
|
||||
private val subtypes = mutableMapOf<TypeRef, MutableSet<TypeRef>>()
|
||||
private val subtypes: MutableMap<TypeRef, MutableSet<TypeRef>> = mutableMapOf()
|
||||
|
||||
fun deleteDataDir() {
|
||||
outputDir.resolve("data").deleteRecursively()
|
||||
}
|
||||
|
||||
fun generate(packages: List<PackageData>) {
|
||||
suspend fun generate(packages: List<PackageData>) {
|
||||
collectData(packages)
|
||||
writeData(packages)
|
||||
}
|
||||
|
||||
private fun collectData(packages: List<PackageData>) {
|
||||
for (pkg in packages) {
|
||||
packageVersions.add(pkg.ref.pkg, pkg.ref.version)
|
||||
packageVersions.add(pkg.ref.pkg, pkg.ref)
|
||||
for (dependency in pkg.dependencies) {
|
||||
if (dependency.isStdlib()) continue
|
||||
// Every package implicitly depends on the stdlib. Showing this dependency adds unwanted
|
||||
@@ -53,187 +53,168 @@ internal class RuntimeDataGenerator(
|
||||
packageUsages.add(dependency.ref, pkg.ref)
|
||||
}
|
||||
for (module in pkg.modules) {
|
||||
moduleVersions.add(module.ref.id, module.ref.version)
|
||||
if (module.moduleClass != null) {
|
||||
collectData(module.moduleClass)
|
||||
}
|
||||
for (clazz in module.classes) {
|
||||
for (clazz in module.effectiveClasses) {
|
||||
collectData(clazz)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// only collect type use information when belonging to the same package.
|
||||
private fun collectData(clazz: ClassData) {
|
||||
classVersions.add(clazz.ref.id, clazz.ref.version)
|
||||
for (superclass in clazz.superclasses) {
|
||||
subtypes.add(superclass, clazz.ref)
|
||||
if (superclass.isInSamePackageAs(clazz.ref)) {
|
||||
subtypes.add(superclass, clazz.ref)
|
||||
}
|
||||
}
|
||||
for (type in clazz.usedTypes) {
|
||||
typeUsages.add(type, clazz.ref)
|
||||
if (type.isInSamePackageAs(clazz.ref)) {
|
||||
typeUsages.add(type, clazz.ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun writeData(packages: List<PackageData>) {
|
||||
for (pkg in packages) {
|
||||
writePackageFile(pkg.ref)
|
||||
for (mod in pkg.modules) {
|
||||
writeModuleFile(mod.ref)
|
||||
for (clazz in mod.classes) {
|
||||
writeClassFile(clazz.ref)
|
||||
val writtenFiles = mutableSetOf<Path>()
|
||||
|
||||
private suspend fun writeData(packages: List<PackageData>) {
|
||||
coroutineScope {
|
||||
for (pkg in packages) {
|
||||
launch { writePackageFilePerVersion(pkg) }
|
||||
for (module in pkg.modules) {
|
||||
for (clazz in module.effectiveClasses) {
|
||||
launch { writeClassFilePerVersion(clazz) }
|
||||
}
|
||||
}
|
||||
}
|
||||
for (pkg in packages.distinctBy { it.ref.pkg }) {
|
||||
launch { writePackageFile(pkg) }
|
||||
for (module in pkg.modules) {
|
||||
for (clazz in module.effectiveClasses) {
|
||||
launch { writeClassFile(clazz) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
updateKnownUsages(packages)
|
||||
}
|
||||
|
||||
private val ModuleData.effectiveClasses: List<ClassData>
|
||||
get() =
|
||||
when (moduleClass) {
|
||||
null -> classes
|
||||
else ->
|
||||
buildList {
|
||||
add(moduleClass)
|
||||
addAll(classes)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* It's possible that a new package uses types/packages from things that are already part of the
|
||||
* docsite.
|
||||
*
|
||||
* If so, update the known usages of those things.
|
||||
*/
|
||||
private suspend fun updateKnownUsages(packages: List<PackageData>) = coroutineScope {
|
||||
val newlyWrittenPackageRefs = packages.mapTo(mutableSetOf()) { it.ref }
|
||||
val existingPackagesWithUpdatedKnownUsages =
|
||||
packageUsages.keys.filterNot { newlyWrittenPackageRefs.contains(it) }
|
||||
for (ref in existingPackagesWithUpdatedKnownUsages) {
|
||||
launch {
|
||||
val runtimeDataPath = outputDir.resolve(ref.perPackageVersionRuntimeDataPath)
|
||||
// we must not have this package in our docsite.
|
||||
if (!runtimeDataPath.isRegularFile()) return@launch
|
||||
val usages = packageUsages[ref]
|
||||
val (data, hasNewData) =
|
||||
ref.existingPerPackageVersionRuntimeData.addKnownUsages(
|
||||
ref,
|
||||
usages,
|
||||
PackageRef::pkg,
|
||||
descendingVersionComparator,
|
||||
)
|
||||
if (hasNewData) {
|
||||
data.doWriteTo(outputDir.resolve(ref.perPackageVersionRuntimeDataPath))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun writePackageFile(ref: PackageRef) {
|
||||
outputDir
|
||||
.resolve("data/${ref.pkg.pathEncoded}/${ref.version.pathEncoded}/index.js")
|
||||
.jsonWriter()
|
||||
.use { writer ->
|
||||
writer.isLenient = true
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_VERSIONS,
|
||||
packageVersions.getOrDefault(ref.pkg, setOf()).sortedWith(descendingVersionComparator),
|
||||
{ it },
|
||||
{ if (it == ref.version) null else ref.copy(version = it).pageUrlRelativeTo(ref) },
|
||||
{ if (it == ref.version) CssConstants.CURRENT_VERSION else null },
|
||||
)
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_USAGES,
|
||||
packageUsages.getOrDefault(ref, setOf()).packagesWithHighestVersions().sortedBy {
|
||||
it.pkg
|
||||
},
|
||||
PackageRef::pkg,
|
||||
{ it.pageUrlRelativeTo(ref) },
|
||||
{ null },
|
||||
)
|
||||
}
|
||||
private val ElementRef<*>.existingPerPackageRuntimeData: RuntimeData by lazyWithReceiver {
|
||||
val path = outputDir.resolve(perPackageRuntimeDataPath)
|
||||
RuntimeData.readOrEmpty(path)
|
||||
}
|
||||
|
||||
private fun writeModuleFile(ref: ModuleRef) {
|
||||
outputDir
|
||||
.resolve(
|
||||
"data/${ref.pkg.pathEncoded}/${ref.version.pathEncoded}/${ref.module.pathEncoded}/index.js"
|
||||
private val ElementRef<*>.existingPerPackageVersionRuntimeData: RuntimeData by lazyWithReceiver {
|
||||
val path = outputDir.resolve(perPackageVersionRuntimeDataPath)
|
||||
RuntimeData.readOrEmpty(path)
|
||||
}
|
||||
|
||||
private fun RuntimeData.doWriteTo(path: Path) {
|
||||
writeTo(path)
|
||||
consoleOut.write("Wrote file ${path.toUri()}\r")
|
||||
}
|
||||
|
||||
private fun RuntimeData.writePerPackageVersion(ref: ElementRef<*>) {
|
||||
val path = outputDir.resolve(ref.perPackageVersionRuntimeDataPath)
|
||||
writtenFiles.add(path)
|
||||
perPackageVersion().writeTo(path)
|
||||
consoleOut.write("Wrote file ${path.toUri()}\r")
|
||||
}
|
||||
|
||||
private fun RuntimeData.writePerPackage(ref: ElementRef<*>) {
|
||||
val path = outputDir.resolve(ref.perPackageRuntimeDataPath)
|
||||
writtenFiles.add(path)
|
||||
perPackage().writeTo(path)
|
||||
consoleOut.write("Wrote file ${path.toUri()}\r")
|
||||
}
|
||||
|
||||
private fun writePackageFile(packageData: PackageData) {
|
||||
val ref = packageData.ref
|
||||
val newVersions = packageVersions[packageData.ref.pkg]?.mapTo(mutableSetOf()) { it.version }
|
||||
val (data, _) =
|
||||
ref.existingPerPackageRuntimeData.addKnownVersions(
|
||||
ref,
|
||||
newVersions,
|
||||
descendingVersionComparator,
|
||||
)
|
||||
.jsonWriter()
|
||||
.use { writer ->
|
||||
writer.isLenient = true
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_VERSIONS,
|
||||
moduleVersions.getOrDefault(ref.id, setOf()).sortedWith(descendingVersionComparator),
|
||||
{ it },
|
||||
{ if (it == ref.version) null else ref.copy(version = it).pageUrlRelativeTo(ref) },
|
||||
{ if (it == ref.version) CssConstants.CURRENT_VERSION else null },
|
||||
)
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_USAGES,
|
||||
typeUsages.getOrDefault(ref.moduleClassRef, setOf()).typesWithHighestVersions().sortedBy {
|
||||
it.displayName
|
||||
},
|
||||
TypeRef::displayName,
|
||||
{ it.pageUrlRelativeTo(ref) },
|
||||
{ null },
|
||||
)
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_SUBTYPES,
|
||||
subtypes.getOrDefault(ref.moduleClassRef, setOf()).typesWithHighestVersions().sortedBy {
|
||||
it.displayName
|
||||
},
|
||||
TypeRef::displayName,
|
||||
{ it.pageUrlRelativeTo(ref) },
|
||||
{ null },
|
||||
)
|
||||
}
|
||||
data.writePerPackage(ref)
|
||||
}
|
||||
|
||||
private fun writeClassFile(ref: TypeRef) {
|
||||
outputDir
|
||||
.resolve(
|
||||
"data/${ref.pkg.pathEncoded}/${ref.version.pathEncoded}/${ref.module.pathEncoded}/${ref.type.pathEncoded}.js"
|
||||
private fun writePackageFilePerVersion(packageData: PackageData) {
|
||||
val ref = packageData.ref
|
||||
val (data, _) =
|
||||
ref.existingPerPackageVersionRuntimeData.addKnownUsages(
|
||||
ref,
|
||||
packageUsages[ref],
|
||||
{ it.pkg },
|
||||
descendingVersionComparator,
|
||||
)
|
||||
.jsonWriter()
|
||||
.use { writer ->
|
||||
writer.isLenient = true
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_VERSIONS,
|
||||
classVersions.getOrDefault(ref.id, setOf()).sortedWith(descendingVersionComparator),
|
||||
{ it },
|
||||
{ if (it == ref.version) null else ref.copy(version = it).pageUrlRelativeTo(ref) },
|
||||
{ if (it == ref.version) CssConstants.CURRENT_VERSION else null },
|
||||
)
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_USAGES,
|
||||
typeUsages.getOrDefault(ref, setOf()).typesWithHighestVersions().sortedBy {
|
||||
it.displayName
|
||||
},
|
||||
TypeRef::displayName,
|
||||
{ it.pageUrlRelativeTo(ref) },
|
||||
{ null },
|
||||
)
|
||||
writer.writeLinks(
|
||||
HtmlConstants.KNOWN_SUBTYPES,
|
||||
subtypes.getOrDefault(ref, setOf()).typesWithHighestVersions().sortedBy {
|
||||
it.displayName
|
||||
},
|
||||
TypeRef::displayName,
|
||||
{ it.pageUrlRelativeTo(ref) },
|
||||
{ null },
|
||||
)
|
||||
}
|
||||
data.writePerPackageVersion(ref)
|
||||
}
|
||||
|
||||
private fun <T> JsonWriter.writeLinks(
|
||||
// HTML element ID
|
||||
id: String,
|
||||
// items based on which links are generated
|
||||
items: List<T>,
|
||||
// link text
|
||||
text: (T) -> String,
|
||||
// link href
|
||||
href: (T) -> String?,
|
||||
// link CSS classes
|
||||
classes: (T) -> String?,
|
||||
) {
|
||||
if (items.isEmpty()) return
|
||||
|
||||
rawText("runtimeData.links('")
|
||||
rawText(id)
|
||||
rawText("','")
|
||||
|
||||
array {
|
||||
for (item in items) {
|
||||
obj {
|
||||
name("text").value(text(item))
|
||||
name("href").value(href(item))
|
||||
name("classes").value(classes(item))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rawText("');\n")
|
||||
private fun writeClassFile(classData: ClassData) {
|
||||
val ref = classData.ref
|
||||
val newVersions = classVersions[ref.id]?.mapTo(mutableSetOf()) { it }
|
||||
val (data, _) =
|
||||
ref.existingPerPackageRuntimeData.addKnownVersions(
|
||||
ref,
|
||||
newVersions,
|
||||
descendingVersionComparator,
|
||||
)
|
||||
data.writePerPackage(ref)
|
||||
}
|
||||
|
||||
private fun Set<PackageRef>.packagesWithHighestVersions(): Collection<PackageRef> {
|
||||
val highestVersions = mutableMapOf<PackageId, PackageRef>()
|
||||
for (ref in this) {
|
||||
val prev = highestVersions[ref.pkg]
|
||||
if (prev == null || descendingVersionComparator.compare(prev.version, ref.version) > 0) {
|
||||
highestVersions[ref.pkg] = ref
|
||||
}
|
||||
}
|
||||
return highestVersions.values
|
||||
}
|
||||
|
||||
private fun Set<TypeRef>.typesWithHighestVersions(): Collection<TypeRef> {
|
||||
val highestVersions = mutableMapOf<TypeId, TypeRef>()
|
||||
for (ref in this) {
|
||||
val prev = highestVersions[ref.id]
|
||||
if (prev == null || descendingVersionComparator.compare(prev.version, ref.version) > 0) {
|
||||
highestVersions[ref.id] = ref
|
||||
}
|
||||
}
|
||||
return highestVersions.values
|
||||
private fun writeClassFilePerVersion(classData: ClassData) {
|
||||
val ref = classData.ref
|
||||
val newSubtypes = subtypes[ref]
|
||||
val (data, _) =
|
||||
ref.existingPerPackageVersionRuntimeData.addKnownSubtypes(
|
||||
ref,
|
||||
newSubtypes,
|
||||
descendingVersionComparator,
|
||||
)
|
||||
data.writePerPackageVersion(ref)
|
||||
}
|
||||
|
||||
private fun <K, V> MutableMap<K, MutableSet<V>>.add(key: K, value: V) {
|
||||
|
||||
@@ -15,166 +15,298 @@
|
||||
*/
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.OutputStream
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.bufferedWriter
|
||||
import kotlin.io.path.createParentDirectories
|
||||
import kotlin.io.path.isRegularFile
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.serialization.KSerializer
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.descriptors.PrimitiveKind
|
||||
import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
|
||||
import kotlinx.serialization.descriptors.SerialDescriptor
|
||||
import kotlinx.serialization.encoding.Decoder
|
||||
import kotlinx.serialization.encoding.Encoder
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.pkl.commons.readString
|
||||
import org.pkl.commons.writeString
|
||||
import org.pkl.core.Member
|
||||
import org.pkl.core.PClass
|
||||
import org.pkl.core.PClass.Method
|
||||
import org.pkl.core.PClass.Property
|
||||
import org.pkl.core.PClassInfo
|
||||
import org.pkl.core.PType
|
||||
import org.pkl.core.util.json.JsonWriter
|
||||
import org.pkl.core.TypeAlias
|
||||
|
||||
internal class SearchIndexGenerator(private val outputDir: Path) {
|
||||
fun generateSiteIndex(packagesData: List<PackageData>) {
|
||||
val path = outputDir.resolve("search-index.js").createParentDirectories()
|
||||
path.jsonWriter().use { writer ->
|
||||
writer.apply {
|
||||
// provide data as JSON string rather than JS literal (more flexible and secure)
|
||||
rawText("searchData='")
|
||||
array {
|
||||
for (pkg in packagesData) {
|
||||
val pkgPath = "${pkg.ref.pkg}/current"
|
||||
obj {
|
||||
name("name").value(pkg.ref.pkg)
|
||||
name("kind").value(0)
|
||||
name("url").value("$pkgPath/index.html")
|
||||
if (pkg.deprecation != null) {
|
||||
name("deprecated").value(true)
|
||||
}
|
||||
}
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
internal class SearchIndexGenerator(private val outputDir: Path, consoleOut: OutputStream) :
|
||||
AbstractGenerator(consoleOut) {
|
||||
companion object {
|
||||
private const val PREFIX = "searchData='"
|
||||
private const val POSTFIX = "';\n"
|
||||
|
||||
for (module in pkg.modules) {
|
||||
obj {
|
||||
name("name").value(module.ref.fullName)
|
||||
name("kind").value(1)
|
||||
name("url").value("$pkgPath/${module.ref.module}/index.html")
|
||||
if (module.deprecation != null) {
|
||||
name("deprecated").value(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
val json = Json {
|
||||
prettyPrint = false
|
||||
explicitNulls = false
|
||||
}
|
||||
}
|
||||
|
||||
private object KindSerializer : KSerializer<Kind> {
|
||||
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("Kind", PrimitiveKind.INT)
|
||||
|
||||
override fun serialize(encoder: Encoder, value: Kind) {
|
||||
encoder.encodeInt(value.value)
|
||||
}
|
||||
|
||||
override fun deserialize(decoder: Decoder): Kind {
|
||||
val intValue = decoder.decodeInt()
|
||||
return Kind.fromInt(intValue)
|
||||
}
|
||||
}
|
||||
|
||||
@Serializable(with = KindSerializer::class)
|
||||
enum class Kind(val value: Int) {
|
||||
PACKAGE(0),
|
||||
MODULE(1),
|
||||
TYPEALIAS(2),
|
||||
CLASS(3),
|
||||
METHOD(4),
|
||||
PROPERTY(5);
|
||||
|
||||
companion object {
|
||||
fun fromInt(value: Int) =
|
||||
entries.firstOrNull { it.value == value }
|
||||
?: throw IllegalArgumentException("Unknown Kind value: $value")
|
||||
}
|
||||
}
|
||||
|
||||
private val searchIndexFile = outputDir.resolve("search-index.js")
|
||||
|
||||
@Serializable
|
||||
data class SearchIndexEntry(
|
||||
val name: String,
|
||||
val kind: Kind,
|
||||
val url: String,
|
||||
val sig: String? = null,
|
||||
val parId: Int? = null,
|
||||
val deprecated: Boolean? = null,
|
||||
val aka: List<String>? = null,
|
||||
)
|
||||
|
||||
data class PackageIndexEntry(
|
||||
val packageEntry: SearchIndexEntry,
|
||||
val moduleEntries: List<SearchIndexEntry>,
|
||||
)
|
||||
|
||||
private fun List<SearchIndexEntry>.writeTo(path: Path) {
|
||||
val self = this
|
||||
val text = buildString {
|
||||
append(PREFIX)
|
||||
append(json.encodeToString(self))
|
||||
append(POSTFIX)
|
||||
}
|
||||
path.writeString(text)
|
||||
writeOutput("Wrote file ${path.toUri()}\r")
|
||||
}
|
||||
|
||||
private fun PackageData.toEntry(): SearchIndexEntry {
|
||||
val pkgPath = "${ref.pkg.pathEncoded}/current"
|
||||
return SearchIndexEntry(
|
||||
name = ref.pkg,
|
||||
kind = Kind.PACKAGE,
|
||||
url = "$pkgPath/${ref.packageRelativeHtmlPath}",
|
||||
deprecated = deprecation?.let { true },
|
||||
)
|
||||
}
|
||||
|
||||
private fun ModuleData.toEntry(basePath: String): SearchIndexEntry {
|
||||
return SearchIndexEntry(
|
||||
name = ref.fullName,
|
||||
kind = Kind.MODULE,
|
||||
url = "$basePath/${ref.packageRelativeHtmlPath}",
|
||||
deprecated = deprecation?.let { true },
|
||||
)
|
||||
}
|
||||
|
||||
private fun DocModule.toEntry(): SearchIndexEntry {
|
||||
val moduleSchema = schema
|
||||
return SearchIndexEntry(
|
||||
name = moduleSchema.moduleName,
|
||||
kind = Kind.MODULE,
|
||||
url = "$path/index.html",
|
||||
)
|
||||
.withAnnotations(moduleSchema.moduleClass)
|
||||
}
|
||||
|
||||
private fun Property.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
|
||||
return SearchIndexEntry(
|
||||
name = simpleName,
|
||||
kind = Kind.PROPERTY,
|
||||
url = "$basePath#$simpleName",
|
||||
sig = renderSignature(this),
|
||||
parId = parentId,
|
||||
)
|
||||
.withAnnotations(this)
|
||||
}
|
||||
|
||||
private fun Method.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
|
||||
return SearchIndexEntry(
|
||||
name = simpleName,
|
||||
kind = Kind.METHOD,
|
||||
url = "$basePath#${simpleName.pathEncoded}()",
|
||||
sig = renderSignature(this),
|
||||
parId = parentId,
|
||||
)
|
||||
.withAnnotations(this)
|
||||
}
|
||||
|
||||
private fun PClass.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
|
||||
return SearchIndexEntry(
|
||||
name = simpleName,
|
||||
kind = Kind.CLASS,
|
||||
url = "$basePath/${simpleName.pathEncoded}.html",
|
||||
parId = parentId,
|
||||
)
|
||||
.withAnnotations(this)
|
||||
}
|
||||
|
||||
private fun TypeAlias.toEntry(parentId: Int, basePath: String): SearchIndexEntry {
|
||||
return SearchIndexEntry(
|
||||
name = simpleName,
|
||||
kind = Kind.TYPEALIAS,
|
||||
url = "$basePath#${simpleName.pathEncoded}",
|
||||
parId = parentId,
|
||||
)
|
||||
.withAnnotations(this)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun SearchIndexEntry.withAnnotations(member: Member?): SearchIndexEntry {
|
||||
if (member == null) return this
|
||||
val deprecatedAnnotation = member.annotations.find { it.classInfo == PClassInfo.Deprecated }
|
||||
val alsoKnownAs = member.annotations.find { it.classInfo == PClassInfo.AlsoKnownAs }
|
||||
return copy(
|
||||
deprecated = deprecatedAnnotation?.let { true },
|
||||
aka = alsoKnownAs?.let { it["names"] as List<String> },
|
||||
)
|
||||
}
|
||||
|
||||
internal fun getCurrentSearchIndex(): List<PackageIndexEntry> {
|
||||
if (!searchIndexFile.isRegularFile()) {
|
||||
return emptyList()
|
||||
}
|
||||
val text = searchIndexFile.readString()
|
||||
if (!(text.startsWith(PREFIX) && text.endsWith(POSTFIX))) {
|
||||
writeOutputLine(
|
||||
"[error] Incorrect existing search-index.js; either doesnt start with prefix '$PREFIX', or end with postfix '$POSTFIX'"
|
||||
)
|
||||
return emptyList()
|
||||
}
|
||||
val jsonStr = text.substring(PREFIX.length, text.length - POSTFIX.length)
|
||||
val entries = json.decodeFromString<List<SearchIndexEntry>>(jsonStr)
|
||||
return buildList {
|
||||
var i = 0
|
||||
|
||||
while (i < entries.size) {
|
||||
val packageEntry = entries[i]
|
||||
i++
|
||||
val moduleEntries = buildList {
|
||||
while (i < entries.size && entries[i].kind == Kind.MODULE) {
|
||||
add(entries[i])
|
||||
i++
|
||||
}
|
||||
}
|
||||
rawText("';\n")
|
||||
add(PackageIndexEntry(packageEntry = packageEntry, moduleEntries = moduleEntries))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun buildSearchIndex(packages: List<PackageData>): List<PackageIndexEntry> = buildList {
|
||||
for (pkg in packages) {
|
||||
val pkgPath = "${pkg.ref.pkg.pathEncoded}/current"
|
||||
add(
|
||||
PackageIndexEntry(
|
||||
packageEntry = pkg.toEntry(),
|
||||
moduleEntries = pkg.modules.map { it.toEntry(basePath = pkgPath) },
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads the current site index, and adds the set of newly generated packages to the index. */
|
||||
fun generateSiteIndex(currentPackages: List<PackageData>) {
|
||||
val searchIndex = buildSearchIndex(currentPackages)
|
||||
searchIndexFile.createParentDirectories()
|
||||
val entries = buildList {
|
||||
for (packageIndexEntry in searchIndex) {
|
||||
add(packageIndexEntry.packageEntry)
|
||||
for (module in packageIndexEntry.moduleEntries) {
|
||||
add(module)
|
||||
}
|
||||
}
|
||||
}
|
||||
entries.writeTo(searchIndexFile)
|
||||
}
|
||||
|
||||
fun generate(docPackage: DocPackage) {
|
||||
val path =
|
||||
outputDir
|
||||
.resolve("${docPackage.name.pathEncoded}/${docPackage.version}/search-index.js")
|
||||
.createParentDirectories()
|
||||
JsonWriter(path.bufferedWriter()).use { writer ->
|
||||
writer.apply {
|
||||
serializeNulls = false
|
||||
// provide data as JSON string rather than JS literal (more flexible and secure)
|
||||
rawText("searchData='")
|
||||
var nextId = 0
|
||||
array {
|
||||
for (docModule in docPackage.docModules) {
|
||||
if (docModule.isUnlisted) continue
|
||||
val entries = buildList {
|
||||
var nextId = 0
|
||||
for (docModule in docPackage.docModules) {
|
||||
if (docModule.isUnlisted) continue
|
||||
|
||||
val module = docModule.schema
|
||||
val moduleId = nextId
|
||||
val module = docModule.schema
|
||||
val moduleId = nextId
|
||||
|
||||
nextId += 1
|
||||
add(docModule.toEntry())
|
||||
val moduleBasePath = docModule.path
|
||||
|
||||
for ((_, property) in module.moduleClass.properties) {
|
||||
if (property.isUnlisted) continue
|
||||
nextId += 1
|
||||
add(property.toEntry(parentId = moduleId, basePath = "${moduleBasePath}/index.html"))
|
||||
}
|
||||
for ((_, method) in module.moduleClass.methods) {
|
||||
if (method.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
add(method.toEntry(parentId = moduleId, basePath = "${moduleBasePath}/index.html"))
|
||||
}
|
||||
for ((_, clazz) in module.classes) {
|
||||
if (clazz.isUnlisted) continue
|
||||
|
||||
val classId = nextId
|
||||
|
||||
nextId += 1
|
||||
add(clazz.toEntry(parentId = moduleId, basePath = moduleBasePath))
|
||||
val classBasePath = "${docModule.path}/${clazz.simpleName}.html"
|
||||
|
||||
for ((_, property) in clazz.properties) {
|
||||
if (property.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(module.moduleName)
|
||||
name("kind").value(1)
|
||||
name("url").value("${docModule.path}/index.html")
|
||||
writeAnnotations(module.moduleClass)
|
||||
}
|
||||
add(property.toEntry(parentId = classId, basePath = classBasePath))
|
||||
}
|
||||
|
||||
for ((propertyName, property) in module.moduleClass.properties) {
|
||||
if (property.isUnlisted) continue
|
||||
for ((_, method) in clazz.methods) {
|
||||
if (method.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(propertyName)
|
||||
name("kind").value(5)
|
||||
name("url").value("${docModule.path}/index.html#$propertyName")
|
||||
name("sig").value(renderSignature(property))
|
||||
name("parId").value(moduleId)
|
||||
writeAnnotations(property)
|
||||
}
|
||||
}
|
||||
|
||||
for ((methodName, method) in module.moduleClass.methods) {
|
||||
if (method.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(methodName)
|
||||
name("kind").value(4)
|
||||
name("url").value("${docModule.path}/index.html#$methodName()")
|
||||
name("sig").value(renderSignature(method))
|
||||
name("parId").value(moduleId)
|
||||
writeAnnotations(method)
|
||||
}
|
||||
}
|
||||
|
||||
for ((className, clazz) in module.classes) {
|
||||
if (clazz.isUnlisted) continue
|
||||
|
||||
val classId = nextId
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(className)
|
||||
name("kind").value(3)
|
||||
name("url").value("${docModule.path}/$className.html")
|
||||
name("parId").value(moduleId)
|
||||
writeAnnotations(clazz)
|
||||
}
|
||||
|
||||
for ((propertyName, property) in clazz.properties) {
|
||||
if (property.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(propertyName)
|
||||
name("kind").value(5)
|
||||
name("url").value("${docModule.path}/$className.html#$propertyName")
|
||||
name("sig").value(renderSignature(property))
|
||||
name("parId").value(classId)
|
||||
writeAnnotations(property)
|
||||
}
|
||||
}
|
||||
|
||||
for ((methodName, method) in clazz.methods) {
|
||||
if (method.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(methodName)
|
||||
name("kind").value(4)
|
||||
name("url").value("${docModule.path}/$className.html#$methodName()")
|
||||
name("sig").value(renderSignature(method))
|
||||
name("parId").value(classId)
|
||||
writeAnnotations(method)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for ((typeAliasName, typeAlias) in module.typeAliases) {
|
||||
if (typeAlias.isUnlisted) continue
|
||||
|
||||
nextId += 1
|
||||
obj {
|
||||
name("name").value(typeAliasName)
|
||||
name("kind").value(2)
|
||||
name("url").value("${docModule.path}/index.html#$typeAliasName")
|
||||
name("parId").value(moduleId)
|
||||
writeAnnotations(typeAlias)
|
||||
}
|
||||
}
|
||||
nextId += 1
|
||||
add(method.toEntry(parentId = classId, basePath = classBasePath))
|
||||
}
|
||||
}
|
||||
rawText("';\n")
|
||||
|
||||
for ((_, typeAlias) in module.typeAliases) {
|
||||
nextId += 1
|
||||
add(typeAlias.toEntry(parentId = moduleId, basePath = "${moduleBasePath}/index.html"))
|
||||
}
|
||||
}
|
||||
}
|
||||
entries.writeTo(path)
|
||||
}
|
||||
|
||||
private fun renderSignature(method: Method): String =
|
||||
@@ -267,24 +399,4 @@ internal class SearchIndexGenerator(private val outputDir: Path) {
|
||||
else -> throw AssertionError("Unknown PType: $type")
|
||||
}
|
||||
}
|
||||
|
||||
private fun JsonWriter.writeAnnotations(member: Member?): JsonWriter {
|
||||
if (member == null) return this
|
||||
|
||||
if (member.annotations.any { it.classInfo == PClassInfo.Deprecated }) {
|
||||
name("deprecated")
|
||||
value(true)
|
||||
}
|
||||
|
||||
member.annotations
|
||||
.find { it.classInfo == PClassInfo.AlsoKnownAs }
|
||||
?.let { alsoKnownAs ->
|
||||
name("aka")
|
||||
array {
|
||||
@Suppress("UNCHECKED_CAST") for (name in alsoKnownAs["names"] as List<String>) value(name)
|
||||
}
|
||||
}
|
||||
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,22 +16,20 @@
|
||||
package org.pkl.doc
|
||||
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
import java.net.URI
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.bufferedWriter
|
||||
import java.nio.file.StandardCopyOption
|
||||
import kotlin.io.path.createParentDirectories
|
||||
import kotlin.io.path.outputStream
|
||||
import org.pkl.core.*
|
||||
import org.pkl.core.util.IoUtils
|
||||
import org.pkl.core.util.json.JsonWriter
|
||||
import org.pkl.parser.Lexer
|
||||
|
||||
// overwrites any existing file
|
||||
internal fun copyResource(resourceName: String, targetDir: Path) {
|
||||
val targetFile = targetDir.resolve(resourceName).apply { createParentDirectories() }
|
||||
getResourceAsStream(resourceName).use { sourceStream ->
|
||||
targetFile.outputStream().use { targetStream -> sourceStream.copyTo(targetStream) }
|
||||
}
|
||||
Files.copy(getResourceAsStream(resourceName), targetFile, StandardCopyOption.REPLACE_EXISTING)
|
||||
}
|
||||
|
||||
internal fun getResourceAsStreamOrNull(resourceName: String): InputStream? =
|
||||
@@ -92,23 +90,6 @@ internal fun getDocCommentOverflow(docComment: String?): String? {
|
||||
}
|
||||
}
|
||||
|
||||
internal fun Path.jsonWriter(): JsonWriter {
|
||||
createParentDirectories()
|
||||
return JsonWriter(bufferedWriter()).apply { serializeNulls = false }
|
||||
}
|
||||
|
||||
internal inline fun JsonWriter.obj(body: JsonWriter.() -> Unit) {
|
||||
beginObject()
|
||||
body()
|
||||
endObject()
|
||||
}
|
||||
|
||||
internal inline fun JsonWriter.array(body: JsonWriter.() -> Unit) {
|
||||
beginArray()
|
||||
body()
|
||||
endArray()
|
||||
}
|
||||
|
||||
internal fun String.replaceSourceCodePlaceholders(
|
||||
path: String,
|
||||
sourceLocation: Member.SourceLocation,
|
||||
@@ -150,3 +131,11 @@ internal val String.asIdentifier: String
|
||||
|
||||
internal val String.pathEncoded
|
||||
get(): String = IoUtils.encodePath(this)
|
||||
|
||||
fun OutputStream.write(str: String) = write(str.toByteArray(Charsets.UTF_8))
|
||||
|
||||
fun OutputStream.writeLine(str: String) = write((str + "\n").toByteArray(Charsets.UTF_8))
|
||||
|
||||
operator fun <A, B> org.pkl.core.util.Pair<A, B>.component1(): A = first
|
||||
|
||||
operator fun <A, B> org.pkl.core.util.Pair<A, B>.component2(): B = second
|
||||
|
||||
@@ -689,30 +689,49 @@ function clearSearch() {
|
||||
updateSearchResults(null);
|
||||
}
|
||||
|
||||
const updateRuntimeDataWith = (buildAnchor) => (fragmentId, entries) => {
|
||||
if (!entries) return;
|
||||
const fragment = document.createDocumentFragment();
|
||||
let first = true;
|
||||
for (const entry of entries) {
|
||||
const a = document.createElement("a");
|
||||
buildAnchor(entry, a);
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
fragment.append(", ");
|
||||
}
|
||||
fragment.append(a);
|
||||
}
|
||||
|
||||
const element = document.getElementById(fragmentId);
|
||||
element.append(fragment);
|
||||
element.classList.remove("hidden"); // dd
|
||||
element.previousElementSibling.classList.remove("hidden"); // dt
|
||||
}
|
||||
|
||||
// Functions called by JS data scripts.
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
const runtimeData = {
|
||||
links: (id, linksJson) => {
|
||||
const links = JSON.parse(linksJson);
|
||||
const fragment = document.createDocumentFragment();
|
||||
let first = true;
|
||||
for (const link of links) {
|
||||
const {text, href, classes} = link
|
||||
const a = document.createElement("a");
|
||||
a.textContent = text;
|
||||
if (href) a.href = href;
|
||||
if (classes) a.className = classes;
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
fragment.append(", ");
|
||||
knownVersions: (versions, myVersion) => {
|
||||
updateRuntimeDataWith((entry, anchor) => {
|
||||
const { text, href } = entry;
|
||||
anchor.textContent = text;
|
||||
// noinspection JSUnresolvedReference
|
||||
if (text === myVersion) {
|
||||
anchor.className = "current-version";
|
||||
} else if (href) {
|
||||
anchor.href = href;
|
||||
}
|
||||
fragment.append(a);
|
||||
})("known-versions", versions);
|
||||
},
|
||||
knownUsagesOrSubtypes: updateRuntimeDataWith((entry, anchor) => {
|
||||
const { text, href } = entry;
|
||||
anchor.textContent = text;
|
||||
// noinspection JSUnresolvedReference
|
||||
anchor.textContent = text;
|
||||
if (href) {
|
||||
anchor.href = href;
|
||||
}
|
||||
|
||||
const element = document.getElementById(id);
|
||||
element.append(fragment);
|
||||
element.classList.remove("hidden"); // dd
|
||||
element.previousElementSibling.classList.remove("hidden"); // dt
|
||||
}
|
||||
}),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user