diff --git a/scalabot-app/build.gradle.kts b/scalabot-app/build.gradle.kts index e841fd1..61cbd82 100644 --- a/scalabot-app/build.gradle.kts +++ b/scalabot-app/build.gradle.kts @@ -13,7 +13,14 @@ dependencies { implementation("io.github.microutils:kotlin-logging:2.1.21") implementation("ch.qos.logback:logback-classic:1.2.10") - implementation("org.eclipse.aether:aether-api:1.1.0") + val aetherVersion = "1.1.0" + implementation("org.eclipse.aether:aether-api:$aetherVersion") + implementation("org.eclipse.aether:aether-util:$aetherVersion") + implementation("org.eclipse.aether:aether-impl:$aetherVersion") + implementation("org.eclipse.aether:aether-transport-file:$aetherVersion") + implementation("org.eclipse.aether:aether-transport-http:$aetherVersion") + implementation("org.eclipse.aether:aether-connector-basic:$aetherVersion") + implementation("org.apache.maven:maven-aether-provider:3.3.9") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0") implementation("com.google.code.gson:gson:2.9.0") diff --git a/scalabot-app/src/main/kotlin/ExtensionComponents.kt b/scalabot-app/src/main/kotlin/ExtensionComponents.kt index 73e324a..46cb00e 100644 --- a/scalabot-app/src/main/kotlin/ExtensionComponents.kt +++ b/scalabot-app/src/main/kotlin/ExtensionComponents.kt @@ -4,6 +4,7 @@ import mu.KotlinLogging import net.lamgc.scalabot.extension.BotExtensionFactory import net.lamgc.scalabot.util.getPriority import org.eclipse.aether.artifact.Artifact +import org.eclipse.aether.repository.LocalRepository import org.telegram.abilitybots.api.util.AbilityExtension import java.io.File import java.io.FileNotFoundException @@ -23,7 +24,8 @@ internal class ExtensionLoader( private val finders: Set = setOf( FileNameFinder, - MavenMetaInformationFinder + MavenMetaInformationFinder, + MavenRepositoryExtensionFinder(LocalRepository("${System.getProperty("user.home")}/.m2/repository")) ) fun getExtensions(): Set { @@ -39,8 +41,12 @@ internal class ExtensionLoader( continue } - val files = getExtensionFiles(filterHighPriorityResult(extensionFilesMap)) - extensionEntries.addAll(getExtensionFactories(extensionArtifact, files.first())) + extensionEntries.addAll( + getExtensionFactories( + extensionArtifact, + filterHighPriorityResult(extensionFilesMap) + ) + ) } return extensionEntries.toSet() } @@ -73,19 +79,13 @@ internal class ExtensionLoader( return foundResult.filterKeys { it.getPriority() == highPriority } } - private fun getExtensionFiles(packageMap: Map>): Set { - val files = mutableSetOf() - for (set in packageMap.values) { - for (foundedExtensionPackage in set) { - files.add(foundedExtensionPackage.loadExtension()) - } - } - return files - } - - private fun getExtensionFactories(extensionArtifact: Artifact, extensionFile: File): Set { + private fun getExtensionFactories( + extensionArtifact: Artifact, + foundResult: Map> + ): Set { + val foundPackage = foundResult.values.first().first() val extClassLoader = - ExtensionClassLoaderCleaner.getOrCreateExtensionClassLoader(extensionArtifact, extensionFile) + ExtensionClassLoaderCleaner.getOrCreateExtensionClassLoader(extensionArtifact, foundPackage) val factories = mutableSetOf() for (factory in extClassLoader.serviceLoader) { try { @@ -125,9 +125,13 @@ internal class ExtensionLoader( if (!checkExtensionPackageFinder(finder)) { continue } - val artifacts = finder.findByArtifact(extensionArtifact, extensionsPath) - if (artifacts.isNotEmpty()) { - result[finder] = artifacts + try { + val artifacts = finder.findByArtifact(extensionArtifact, extensionsPath) + if (artifacts.isNotEmpty()) { + result[finder] = artifacts + } + } catch (e: Exception) { + log.error { "搜索器 ${finder::class.java.name} 在搜索扩展 `$extensionArtifact` 时发生错误:" } } } return result @@ -184,9 +188,12 @@ internal object ExtensionClassLoaderCleaner { private val usageCountMap = mutableMapOf() @Synchronized - fun getOrCreateExtensionClassLoader(extensionArtifact: Artifact, extensionFile: File): ExtensionClassLoader { + fun getOrCreateExtensionClassLoader( + extensionArtifact: Artifact, + foundExtensionPackage: FoundExtensionPackage + ): ExtensionClassLoader { return if (!artifactMap.containsKey(extensionArtifact)) { - val newClassLoader = ExtensionClassLoader(extensionFile) + val newClassLoader = foundExtensionPackage.createClassLoader() artifactMap[extensionArtifact] = newClassLoader usageCountMap[newClassLoader] = AtomicInteger(1) newClassLoader @@ -237,6 +244,14 @@ internal interface ExtensionPackageFinder { * @return 返回按搜索器的方式可以找到的所有与构件坐标有关的扩展包路径. */ fun findByArtifact(extensionArtifact: Artifact, extensionsPath: File): Set + + /** + * 获取类加载器工厂. + * + * 搜索器可根据需求自行实现类加载器工厂. + * @return 返回 [ExtensionClassLoaderFactory] 实现. + */ + fun getClassLoaderFactory(): ExtensionClassLoaderFactory = DefaultExtensionClassLoaderFactory } /** @@ -264,15 +279,18 @@ internal interface FoundExtensionPackage { * 当调用本方法时, Finder 可以将扩展包下载到本地(如果扩展包在远端服务器的话). * @return 返回扩展包在本地存储时指向扩展包文件的 File 对象. */ - fun loadExtension(): File + fun getPackageFile(): File /** - * + * 找到该扩展包的发现器对象. */ fun getExtensionPackageFinder(): ExtensionPackageFinder } +private fun FoundExtensionPackage.createClassLoader(): ExtensionClassLoader = + getExtensionPackageFinder().getClassLoaderFactory().createClassLoader(this) + /** * 已找到的扩展包文件. * @param artifact 扩展包构件坐标. @@ -294,7 +312,7 @@ internal class FileFoundExtensionPackage( override fun getRawUrl(): URL = file.canonicalFile.toURI().toURL() - override fun loadExtension(): File = file + override fun getPackageFile(): File = file override fun getExtensionPackageFinder(): ExtensionPackageFinder = finder } @@ -304,19 +322,38 @@ internal class FileFoundExtensionPackage( * * 通过为每个扩展包提供专有的加载器, 可防止意外使用其他扩展的类(希望如此). * @param urls 扩展包资源 Url. + * @param dependencyLoader 依赖项的类加载器. 当扩展包含有其他依赖项时, 需将依赖项单独设置在一个类加载器中, 以确保扩展加载安全. */ -internal class ExtensionClassLoader(vararg urls: URL) : - URLClassLoader(urls) { +internal class ExtensionClassLoader(urls: Array, dependencyLoader: ClassLoader = getSystemClassLoader()) : + URLClassLoader(urls, dependencyLoader) { /** * 指定扩展包 File 来创建 ClassLoader. * @param extensionFile 扩展包文件. */ constructor(extensionFile: File) : - this(URL(getUrlString(extensionFile))) + this(arrayOf(URL(getUrlString(extensionFile)))) val serviceLoader: ServiceLoader = ServiceLoader.load(BotExtensionFactory::class.java, this) + // 为防止从非扩展包位置引入扩展的问题, 覆写了以下两个方法 + // 当寻找 BotExtensionFactory 时, 将不再遵循双亲委派原则, + // 以免使用了不来自扩展包的机器人扩展. + + override fun getResources(name: String?): Enumeration { + if (BotExtensionFactory::class.java.equals(name)) { + return findResources(name) + } + return super.getResources(name) + } + + override fun getResource(name: String?): URL? { + if (BotExtensionFactory::class.java.equals(name)) { + return findResource(name) + } + return super.getResource(name) + } + companion object { private fun getUrlString(extensionFile: File, defaultScheme: String = "file:///"): String { return when (extensionFile.extension.lowercase()) { @@ -327,6 +364,31 @@ internal class ExtensionClassLoader(vararg urls: URL) : } } +/** + * 扩展类加载器工厂接口. + * + * 可供有多依赖需求的扩展使用, 由 Finder 提供. + */ +internal interface ExtensionClassLoaderFactory { + + /** + * 创建扩展包的类加载器. + * @param foundExtensionPackage 已找到的扩展包信息. + */ + fun createClassLoader(foundExtensionPackage: FoundExtensionPackage): ExtensionClassLoader + +} + +/** + * 针对单文件依赖的 ClassLoader 工厂. + */ +internal object DefaultExtensionClassLoaderFactory : ExtensionClassLoaderFactory { + override fun createClassLoader(foundExtensionPackage: FoundExtensionPackage): ExtensionClassLoader { + return ExtensionClassLoader(foundExtensionPackage.getPackageFile()) + } +} + + /** * 搜索器规则. * @property priority 搜索器优先级. 优先级从 0 (最高)开始, 相同构件坐标下将使用优先级最高的搜索器所找到的文件. diff --git a/scalabot-app/src/main/kotlin/ExtensionFinders.kt b/scalabot-app/src/main/kotlin/ExtensionFinders.kt index c910fe1..f8e08a0 100644 --- a/scalabot-app/src/main/kotlin/ExtensionFinders.kt +++ b/scalabot-app/src/main/kotlin/ExtensionFinders.kt @@ -1,15 +1,33 @@ package net.lamgc.scalabot +import com.google.common.base.Throwables +import mu.KotlinLogging import net.lamgc.scalabot.util.deepListFiles import net.lamgc.scalabot.util.equalsArtifact +import org.apache.maven.repository.internal.MavenRepositorySystemUtils +import org.eclipse.aether.RepositorySystem import org.eclipse.aether.artifact.Artifact import org.eclipse.aether.artifact.DefaultArtifact +import org.eclipse.aether.collection.CollectRequest +import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory +import org.eclipse.aether.graph.Dependency +import org.eclipse.aether.repository.* +import org.eclipse.aether.resolution.ArtifactRequest +import org.eclipse.aether.resolution.ArtifactResult +import org.eclipse.aether.resolution.DependencyRequest +import org.eclipse.aether.spi.connector.RepositoryConnectorFactory +import org.eclipse.aether.spi.connector.transport.TransporterFactory +import org.eclipse.aether.transport.file.FileTransporterFactory +import org.eclipse.aether.transport.http.HttpTransporterFactory +import org.eclipse.aether.util.filter.ScopeDependencyFilter import org.jdom2.Document import org.jdom2.filter.Filters import org.jdom2.input.SAXBuilder import org.jdom2.xpath.XPathFactory import java.io.File import java.io.InputStream +import java.net.URL +import java.net.URLClassLoader import java.util.* import java.util.jar.JarEntry import java.util.jar.JarInputStream @@ -20,14 +38,22 @@ import java.util.jar.JarInputStream * * 将搜索文件名(不带扩展包名)结尾为 `${groupId}_${artifactId}_${version}` 的文件. * 比如说 `(Example Extension) org.example_scalabot-example_v1.0.0-SNAPSHOT.jar` 是可以的 + * + * 使用这种方式, 要求扩展包将依赖打包进自己的 jar, 可能会出现分发包体积较大的情况. */ @FinderRules(priority = FinderPriority.LOCAL) internal object FileNameFinder : ExtensionPackageFinder { + private val log = KotlinLogging.logger { } + + private val allowExtensionNames = setOf("jar", "jmod", "zip") + override fun findByArtifact(extensionArtifact: Artifact, extensionsPath: File): Set { val focusName = getExtensionFilename(extensionArtifact) + log.debug { "扩展 $extensionArtifact 匹配规则: $focusName" } val files = extensionsPath.listFiles { file -> - file.nameWithoutExtension.endsWith(focusName) + file.nameWithoutExtension.endsWith(focusName) && + (allowExtensionNames.contains(file.extension.lowercase()) || file.isDirectory) } ?: return emptySet() val extensionPackage = mutableSetOf() @@ -195,3 +221,247 @@ internal object MavenMetaInformationFinder : ExtensionPackageFinder { !prop.containsKey(key) || prop.getProperty(key).trim().isEmpty() } + +/** + * 从指定的 Maven 仓库下载扩展包. + * + * 注: 也会下载扩展包的依赖包. + * + * 建议扩展包不要将依赖项打包到本体, 搜索器会自动下载依赖项并自动缓存, 这对于后续更新来讲十分有用. (也会给其他扩展包共享依赖) + * 目前每个扩展包的依赖都是分开加载的, 我会听取社区意见, 来决定是否有必要让依赖项共享同一个 [ClassLoader]. + * + * 推荐使用这种方式来安装扩展. + */ +@FinderRules(priority = FinderPriority.REMOTE) +internal class MavenRepositoryExtensionFinder( + private val localRepository: LocalRepository, + private val proxy: Proxy? = null, + private val remoteRepositories: List = listOf(getMavenCentralRepository(proxy)), +) : ExtensionPackageFinder { + + private val repositorySystem = createRepositorySystem() + + private val repoSystemSession = createRepositorySystemSession() + + private fun createRepositorySystem() = MavenRepositorySystemUtils.newServiceLocator().apply { + addService(RepositoryConnectorFactory::class.java, BasicRepositoryConnectorFactory::class.java) + addService(TransporterFactory::class.java, FileTransporterFactory::class.java) + addService(TransporterFactory::class.java, HttpTransporterFactory::class.java) + }.getService(RepositorySystem::class.java) + + private fun createRepositorySystemSession() = MavenRepositorySystemUtils.newSession().apply { + localRepositoryManager = repositorySystem.newLocalRepositoryManager( + this, + this@MavenRepositoryExtensionFinder.localRepository + ) + } + + override fun findByArtifact(extensionArtifact: Artifact, extensionsPath: File): Set { + val extensionArtifactResult = repositorySystem.resolveArtifact( + repoSystemSession, + ArtifactRequest(extensionArtifact, remoteRepositories, null) + ) + val extResolvedArtifact = extensionArtifactResult.artifact + if (!extensionArtifactResult.isResolved) { + if (extensionArtifactResult.isMissing) { + log.warn { "在指定的仓库中找不到构件: ${extensionArtifactResult.artifact}" } + } else { + printArtifactResultExceptions(extensionArtifactResult.exceptions) + } + return emptySet() + } + + val request = DependencyRequest( + CollectRequest(Dependency(extResolvedArtifact, null), remoteRepositories), + ScopeDependencyFilter(setOf("runtime", "compile", "provided"), null) + ) + val dependencyResult = repositorySystem.resolveDependencies(repoSystemSession, request) + val dependencies = checkAndCollectDependencyArtifacts(extensionArtifact, dependencyResult.artifactResults) + ?: return emptySet() + + return setOf(MavenExtensionPackage(this, extResolvedArtifact, extensionArtifactResult.repository, dependencies)) + } + + private fun checkAndCollectDependencyArtifacts( + extensionArtifact: Artifact, + dependencyResult: List + ): Set? { + val resolveFailedArtifacts = mutableSetOf() + for (artifactResult in dependencyResult) { + if (!artifactResult.isResolved) { + resolveFailedArtifacts.add(artifactResult) + } + } + + if (resolveFailedArtifacts.isNotEmpty()) { + log.error { + StringBuilder("扩展包 `$extensionArtifact` 下列依赖项获取失败: \n").apply { + resolveFailedArtifacts.forEach { + append("\t- 依赖项 `${it.artifact}` ") + if (it.isMissing) { + append(" 无法从指定仓库中找到.") + } else { + append("\n") + for (e in it.exceptions) { + append("\t\t- ${e::class.java.name}: ${e.message}\n") + } + } + } + }.toString() + } + return null + } + + return dependencyResult.map { + log.debug { "依赖项 ${it.artifact} 文件路径: ${it.artifact.file}" } + it.artifact!! + }.toSet() + } + + private fun printArtifactResultExceptions(exceptions: List) { + log.warn { + val builder = StringBuilder("构件可能已找到, 但由于以下原因导致获取失败: \n") + exceptions.forEachIndexed { index, exception -> + builder.append("[$index] ").append(Throwables.getStackTraceAsString(exception)).append('\n') + } + return@warn builder.toString() + } + } + + @Suppress("unused") + companion object { + @JvmStatic + private val log = KotlinLogging.logger { } + + /** + * 将 [URL] 转换成 [RemoteRepository] 对象. + * @param url 远端仓库地址. + * @param type 仓库布局类型, 如果是 Maven 2 的仓库布局, 则为 `default`, 如果是 Maven 1 的旧版仓库布局, 则为 `legacy`. + * @param proxy 是否使用代理访问仓库, 默认为 `null`. + */ + fun resolveRepositoryByUrl( + url: URL, + repoId: String? = null, + type: String = "default", + proxy: Proxy?, + authentication: Authentication? = null + ): RemoteRepository { + val builder = RemoteRepository.Builder(repoId, type, url.toString()) + if (proxy != null) { + builder.setProxy(proxy) + } + builder.setReleasePolicy( + RepositoryPolicy( + true, + RepositoryPolicy.UPDATE_POLICY_DAILY, + RepositoryPolicy.CHECKSUM_POLICY_WARN + ) + ) + builder.setSnapshotPolicy( + RepositoryPolicy( + true, + RepositoryPolicy.UPDATE_POLICY_ALWAYS, + RepositoryPolicy.CHECKSUM_POLICY_WARN + ) + ) + if (authentication != null) { + builder.setAuthentication(authentication) + } + return builder.build() + } + + /** + * Maven 中央仓库 Url. + */ + @Suppress("MemberVisibilityCanBePrivate") + const val MAVEN_CENTRAL_URL = "https://repo1.maven.org/maven2/" + + /** + * 获取 Maven 中央仓库的 [RemoteRepository] 对象. + */ + fun getMavenCentralRepository(proxy: Proxy? = null): RemoteRepository { + val builder = RemoteRepository.Builder("central", "default", MAVEN_CENTRAL_URL) + if (proxy != null) { + builder.setProxy(proxy) + } + return builder.build() + } + + } + + class MavenExtensionPackage( + private val finder: ExtensionPackageFinder, + private val artifact: Artifact, + private val fromRepository: ArtifactRepository, + val dependencies: Set + ) : FoundExtensionPackage { + override fun getExtensionArtifact(): Artifact = artifact + + override fun getRawUrl(): URL { + return if (fromRepository is RemoteRepository) { + val urlStr = if (fromRepository.url.endsWith("/")) { + fromRepository.url + getArtifactPath(artifact) + } else { + fromRepository.url + "/" + getArtifactPath(artifact) + } + URL(urlStr) + } else { + getPackageFile().toURI().toURL() + } + } + + override fun getPackageFile(): File = artifact.file + + override fun getExtensionPackageFinder() = finder + + /** + * 获取 Artifact 在 Maven 仓库中的路径. + * + * 遵循 [Repository Layout - Final](https://cwiki.apache.org/confluence/display/MAVENOLD/Repository+Layout+-+Final) + */ + private fun getArtifactPath(artifact: Artifact): String { + val groups = artifact.groupId.split('.') + return StringBuilder("/").apply { + for (group in groups) { + append(group).append('/') + } + append("${artifact.artifactId}/${artifact.version}/") + append("${artifact.artifactId}-${artifact.version}") + if (artifact.classifier.isNotEmpty()) { + append("-${artifact.classifier}") + } + append(".${artifact.extension}") + }.toString() + } + + } + + override fun getClassLoaderFactory(): ExtensionClassLoaderFactory { + return object : ExtensionClassLoaderFactory { + override fun createClassLoader(foundExtensionPackage: FoundExtensionPackage): ExtensionClassLoader { + if (foundExtensionPackage !is MavenExtensionPackage) { + throw IllegalArgumentException("Unsupported FoundExtensionPackage type: $foundExtensionPackage") + } + + val urls = mutableSetOf() + for (dependency in foundExtensionPackage.dependencies) { + val dependencyFile = dependency.file ?: continue + urls.add(dependencyFile.toURI().toURL()) + } + + // 将依赖的 ClassLoader 与 ExtensionPackage 的 ClassLoader 分开 + // 这么做可以防范依赖中隐藏的 SPI 注册, 避免安全隐患. + + val dependenciesUrlArray = urls.toTypedArray() + val dependenciesClassLoader = URLClassLoader(dependenciesUrlArray) + + return ExtensionClassLoader( + arrayOf(foundExtensionPackage.getPackageFile().toURI().toURL()), + dependenciesClassLoader + ) + } + } + } + +} + diff --git a/scalabot-app/src/main/kotlin/util/Utils.kt b/scalabot-app/src/main/kotlin/util/Utils.kt index bd3d91c..bbc7b97 100644 --- a/scalabot-app/src/main/kotlin/util/Utils.kt +++ b/scalabot-app/src/main/kotlin/util/Utils.kt @@ -7,6 +7,7 @@ import org.eclipse.aether.artifact.Artifact import java.io.File import java.io.FileFilter import java.io.FilenameFilter +import java.net.URL internal fun ByteArray.toHaxString(): String = ByteUtils.bytesToHexString(this) @@ -93,4 +94,18 @@ private object UtilsInternal { log.debug { "All registered hook resources have been closed." } }, "Shutdown-AutoCloseable")) } -} \ No newline at end of file +} + +fun URL.resolveToFile(canonical: Boolean = true): File { + if ("file" != protocol) { + throw ClassCastException("Only the URL of the `file` protocol can be converted into a File object.") + } + + val urlString = toString().substringAfter(':') + val file = File(urlString) + return if (canonical) { + file.canonicalFile + } else { + file + } +}