Skip to content

Commit dd9cb52

Browse files
martinbonninmartinbonnin
authored
upload in parallel (#180)
1 parent e2a67b9 commit dd9cb52

File tree

7 files changed

+162
-112
lines changed

7 files changed

+162
-112
lines changed

‎gradle/libs.versions.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ gradle-min = "dev.gradleplugins:gradle-api:8.8"
1212
kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test" }
1313
xmlutil = "io.github.pdvrieze.xmlutil:serialization:0.91.1"
1414
kgp = { module = "org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin", version.ref = "kgp" }
15+
coroutines = "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2"
1516

1617
[plugins]
1718
kgp = { id = "org.jetbrains.kotlin.jvm", version.ref = "kgp" }

‎nmcp/api/nmcp.api

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ public abstract class nmcp/CentralPortalOptions {
55
public abstract fun getPublicationName ()Lorg/gradle/api/provider/Property;
66
public abstract fun getPublishingTimeout ()Lorg/gradle/api/provider/Property;
77
public abstract fun getPublishingType ()Lorg/gradle/api/provider/Property;
8+
public abstract fun getUploadSnapshotsParallelism ()Lorg/gradle/api/provider/Property;
89
public abstract fun getUsername ()Lorg/gradle/api/provider/Property;
910
public abstract fun getValidationTimeout ()Lorg/gradle/api/provider/Property;
1011
}
@@ -61,11 +62,11 @@ public final class nmcp/internal/task/NmcpFindDeploymentNameEntryPoint$Companion
6162
public final class nmcp/internal/task/NmcpPublishFileByFileToSnapshotsEntryPoint {
6263
public static final field Companion Lnmcp/internal/task/NmcpPublishFileByFileToSnapshotsEntryPoint$Companion;
6364
public fun <init> ()V
64-
public static final fun run (Ljava/util/function/BiConsumer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/List;)V
65+
public static final fun run (Ljava/util/function/BiConsumer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/List;I)V
6566
}
6667

6768
public final class nmcp/internal/task/NmcpPublishFileByFileToSnapshotsEntryPoint$Companion {
68-
public final fun run (Ljava/util/function/BiConsumer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/List;)V
69+
public final fun run (Ljava/util/function/BiConsumer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/List;I)V
6970
}
7071

7172
public final class nmcp/internal/task/NmcpPublishWithPublisherApiEntryPoint {
@@ -84,6 +85,7 @@ public abstract interface class nmcp/transport/Content {
8485

8586
public final class nmcp/transport/PublishFileByFileKt {
8687
public static final fun publishFileByFile (Lnmcp/transport/Transport;Ljava/util/List;)V
88+
public static final fun publishFileByFile (Lnmcp/transport/Transport;Ljava/util/List;I)V
8789
}
8890

8991
public abstract interface class nmcp/transport/Transport {

‎nmcp/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ gratatouille {
2525
dependencies {
2626
implementation(libs.json)
2727
implementation(libs.okio)
28+
implementation(libs.coroutines)
2829
api(libs.okhttp)
2930
implementation(libs.xmlutil)
3031

‎nmcp/src/main/kotlin/nmcp/CentralPortalOptions.kt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,12 @@ abstract class CentralPortalOptions {
7171
* Default: "https://central.sonatype.com/".
7272
*/
7373
abstract val baseUrl: Property<String>
74+
75+
/**
76+
* The parallelism level for uploading publications to snapshots.
77+
* Inside a publication (a similar group/artifact), files are still uploaded serially.
78+
*
79+
* Default: 1.
80+
*/
81+
abstract val uploadSnapshotsParallelism: Property<Int>
7482
}

‎nmcp/src/main/kotlin/nmcp/internal/task/nmcpPublishFileByFileToSnapshots.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ internal fun nmcpPublishFileByFileToSnapshots(
1414
username: String?,
1515
password: String?,
1616
inputFiles: GInputFiles,
17+
parallelism: Int,
1718
) {
1819
val authorizationHeader = if (username != null) {
1920
check(!password.isNullOrBlank()) {
@@ -36,5 +37,5 @@ internal fun nmcpPublishFileByFileToSnapshots(
3637
)
3738

3839
logger.lifecycle("Nmcp: uploading files to $snapshotsUrl")
39-
publishFileByFile(transport, inputFiles)
40+
publishFileByFile(transport, inputFiles, parallelism)
4041
}

‎nmcp/src/main/kotlin/nmcp/internal/utils.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import org.gradle.api.attributes.HasConfigurableAttributes
1616
import org.gradle.api.attributes.Usage
1717
import org.gradle.api.attributes.Usage.USAGE_ATTRIBUTE
1818
import org.gradle.api.file.FileCollection
19+
import org.gradle.api.provider.Property
1920
import org.gradle.api.publish.plugins.PublishingPlugin.PUBLISH_TASK_GROUP
2021
import org.gradle.api.tasks.bundling.Zip
2122

@@ -120,6 +121,7 @@ internal fun Project.registerPublishToCentralPortalTasks(
120121
password = spec.password,
121122
snapshotsUrl = project.provider { "https://central.sonatype.com/repository/maven-snapshots/" },
122123
inputFiles = inputFiles,
124+
parallelism = spec.uploadSnapshotsParallelism,
123125
)
124126
if (snapshotsLifecycleTaskName != null) {
125127
project.tasks.register(snapshotsLifecycleTaskName) {

‎nmcp/src/main/kotlin/nmcp/transport/publishFileByFile.kt

Lines changed: 144 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ import gratatouille.tasks.FileWithPath
44
import gratatouille.tasks.GInputFiles
55
import java.security.MessageDigest
66
import java.time.Instant
7+
import kotlinx.coroutines.Dispatchers
8+
import kotlinx.coroutines.joinAll
9+
import kotlinx.coroutines.launch
10+
import kotlinx.coroutines.runBlocking
11+
import kotlinx.coroutines.withContext
712
import kotlinx.serialization.decodeFromString
813
import kotlinx.serialization.encodeToString
914
import nmcp.internal.task.ArtifactMetadata
@@ -16,6 +21,14 @@ import okio.ByteString.Companion.toByteString
1621
fun publishFileByFile(
1722
transport: Transport,
1823
inputFiles: GInputFiles,
24+
) {
25+
return publishFileByFile(transport, inputFiles, 1)
26+
}
27+
28+
fun publishFileByFile(
29+
transport: Transport,
30+
inputFiles: GInputFiles,
31+
parallelism: Int,
1932
) {
2033
val allFiles = inputFiles.filter { it.file.isFile }
2134
val gavPaths = allFiles.filter { it.normalizedPath.endsWith(".pom") || it.normalizedPath.endsWith(".module") }
@@ -24,137 +37,151 @@ fun publishFileByFile(
2437

2538
val lastUpdated = timestampNow()
2639

27-
gavPaths.forEach { gavPath ->
28-
val gav = Gav.from(gavPath)
29-
val version = gav.version
30-
val gavFiles = allFiles.filter { it.normalizedPath.startsWith(gavPath) }
31-
32-
/**
33-
* This is a proper directory containing artifacts
34-
*/
35-
if (version.endsWith("-SNAPSHOT")) {
36-
/**
37-
* This is a snapshot:
38-
* - update the [version metadata](https://maven.apache.org/repositories/metadata.html).
39-
* - patch the file names to include the new build number.
40-
*
41-
* See https://s01.oss.sonatype.org/content/repositories/snapshots/com/apollographql/apollo/apollo-api-jvm/maven-metadata.xml for an example.
42-
*
43-
* For snapshots, it's not 100% clear who owns the metadata as the repository might expire some snapshot and therefore need to rewrite the
44-
* metadata to keep things consistent. This means there are 2 possibly concurrent writers to maven-metadata.xml: the repository and the
45-
* publisher. Hopefully, it's not too much of a problem in practice.
46-
*
47-
* See https://github.com/gradle/gradle/blob/d1ee068b1ee7f62ffcbb549352469307781af72e/platforms/software/maven/src/main/java/org/gradle/api/publish/maven/internal/publisher/MavenRemotePublisher.java#L70.
48-
*/
49-
val versionMetadataPath = "$gavPath/maven-metadata.xml"
50-
val localVersionMetadataFile = gavFiles.firstOrNull {
51-
it.normalizedPath == versionMetadataPath
52-
}
53-
val localVersionMetadata = if (localVersionMetadataFile != null) {
54-
xml.decodeFromString<VersionMetadata>(localVersionMetadataFile.file.readText())
55-
} else {
56-
VersionMetadata(
57-
groupId = gav.groupId,
58-
artifactId = gav.artifactId,
59-
version = gav.version,
60-
versioning = VersionMetadata.Versioning(
61-
snapshot = VersionMetadata.Snapshot(timestamp = lastUpdated, buildNumber = 1),
62-
lastUpdated = lastUpdated,
63-
snapshotVersions = emptyList()
64-
)
65-
)
66-
}
67-
68-
val remoteVersionMetadata = transport.get(versionMetadataPath)
69-
70-
val buildNumber = if (remoteVersionMetadata == null) {
71-
1
72-
} else {
73-
xml.decodeFromString<VersionMetadata>(remoteVersionMetadata.use { it.readUtf8() }).versioning.snapshot.buildNumber + 1
74-
}
75-
76-
val newVersionMetadata = localVersionMetadata.copy(
77-
versioning = localVersionMetadata.versioning.copy(
78-
snapshot = localVersionMetadata.versioning.snapshot.copy(buildNumber = buildNumber),
79-
),
80-
)
81-
82-
val renamedFiles = gavFiles.mapNotNull {
83-
if (it.file.name.startsWith("maven-metadata.xml")) {
84-
return@mapNotNull null
40+
runBlocking {
41+
withContext(Dispatchers.IO.limitedParallelism(parallelism)) {
42+
gavPaths.map {
43+
launch {
44+
publishGav(it, allFiles, lastUpdated, transport)
8545
}
86-
val newName = it.file.name.replaceBuildNumber(gav.artifactId, gav.version, buildNumber)
87-
FileWithPath(it.file, "$gavPath/$newName")
88-
}
89-
90-
transport.uploadFiles(renamedFiles)
91-
92-
val bytes = encodeToXml(newVersionMetadata).toByteArray()
93-
transport.put(versionMetadataPath, bytes)
94-
setOf("md5", "sha1", "sha256", "sha512").forEach {
95-
transport.put("$versionMetadataPath.$it", bytes.digest(it.uppercase()))
96-
}
97-
} else {
98-
/**
99-
* Not a snapshot, plainly update all the files
100-
*/
101-
transport.uploadFiles(gavFiles)
46+
}.joinAll()
10247
}
48+
}
49+
}
10350

51+
private fun publishGav(
52+
gavPath: String,
53+
allFiles: List<FileWithPath>,
54+
lastUpdated: String,
55+
transport: Transport,
56+
) {
57+
val gav = Gav.from(gavPath)
58+
val version = gav.version
59+
val gavFiles = allFiles.filter { it.normalizedPath.startsWith(gavPath) }
60+
61+
/**
62+
* This is a proper directory containing artifacts
63+
*/
64+
if (version.endsWith("-SNAPSHOT")) {
10465
/**
105-
* Update the [artifact metadata](https://maven.apache.org/repositories/metadata.html).
66+
* This is a snapshot:
67+
* - update the [version metadata](https://maven.apache.org/repositories/metadata.html).
68+
* - patch the file names to include the new build number.
69+
*
70+
* See https://s01.oss.sonatype.org/content/repositories/snapshots/com/apollographql/apollo/apollo-api-jvm/maven-metadata.xml for an example.
10671
*
107-
* See https://repo1.maven.org/maven2/com/apollographql/apollo/apollo-api-jvm/maven-metadata.xml for an example.
72+
* For snapshots, it's not 100% clear who owns the metadata as the repository might expire some snapshot and therefore need to rewrite the
73+
* metadata to keep things consistent. This means there are 2 possibly concurrent writers to maven-metadata.xml: the repository and the
74+
* publisher. Hopefully, it's not too much of a problem in practice.
75+
*
76+
* See https://github.com/gradle/gradle/blob/d1ee068b1ee7f62ffcbb549352469307781af72e/platforms/software/maven/src/main/java/org/gradle/api/publish/maven/internal/publisher/MavenRemotePublisher.java#L70.
10877
*/
109-
val index = gavPath.lastIndexOf('/')
110-
check (index != -1) {
111-
"Nmcp: invalid gav path: '$gavPath'"
78+
val versionMetadataPath = "$gavPath/maven-metadata.xml"
79+
val localVersionMetadataFile = gavFiles.firstOrNull {
80+
it.normalizedPath == versionMetadataPath
11281
}
113-
val artifactMetadataPath = "${gavPath.substring(0, index)}/maven-metadata.xml"
114-
val localArtifactMetadataFile = allFiles.firstOrNull { it.normalizedPath == artifactMetadataPath }
115-
val localArtifactMetadata = if (localArtifactMetadataFile == null) {
116-
// The publisher did not artifact level metadata, let's
117-
ArtifactMetadata(
82+
val localVersionMetadata = if (localVersionMetadataFile != null) {
83+
xml.decodeFromString<VersionMetadata>(localVersionMetadataFile.file.readText())
84+
} else {
85+
VersionMetadata(
11886
groupId = gav.groupId,
11987
artifactId = gav.artifactId,
120-
versioning = ArtifactMetadata.Versioning(
121-
latest = gav.version,
122-
release = gav.version,
123-
versions = emptyList(),
88+
version = gav.version,
89+
versioning = VersionMetadata.Versioning(
90+
snapshot = VersionMetadata.Snapshot(timestamp = lastUpdated, buildNumber = 1),
12491
lastUpdated = lastUpdated,
125-
)
92+
snapshotVersions = emptyList(),
93+
),
12694
)
127-
} else {
128-
xml.decodeFromString<ArtifactMetadata>(localArtifactMetadataFile.file.readText())
12995
}
13096

131-
val remoteArtifactMetadata = transport.get(artifactMetadataPath)
97+
val remoteVersionMetadata = transport.get(versionMetadataPath)
13298

133-
val existingVersions = if (remoteArtifactMetadata != null) {
134-
xml.decodeFromString<ArtifactMetadata>(remoteArtifactMetadata.use { it.readUtf8() }).versioning.versions
99+
val buildNumber = if (remoteVersionMetadata == null) {
100+
1
135101
} else {
136-
emptyList()
102+
xml.decodeFromString<VersionMetadata>(remoteVersionMetadata.use { it.readUtf8() }).versioning.snapshot.buildNumber + 1
137103
}
138104

139-
/**
140-
* See https://github.com/gradle/gradle/blob/cb0c615fb8e3690971bb7f89ad80f58943360624/platforms/software/maven/src/main/java/org/gradle/api/publish/maven/internal/publisher/AbstractMavenPublisher.java#L116.
141-
*/
142-
val versions = existingVersions.toMutableList()
143-
if (!versions.none { it == gav.version }) {
144-
versions.add(gav.version)
145-
}
146-
val newArtifactMetadata = localArtifactMetadata.copy(
147-
versioning = localArtifactMetadata.versioning.copy(
148-
versions = versions,
105+
val newVersionMetadata = localVersionMetadata.copy(
106+
versioning = localVersionMetadata.versioning.copy(
107+
snapshot = localVersionMetadata.versioning.snapshot.copy(buildNumber = buildNumber),
149108
),
150109
)
151110

152-
val bytes = encodeToXml(newArtifactMetadata).toByteArray()
153-
transport.put(artifactMetadataPath, bytes)
111+
val renamedFiles = gavFiles.mapNotNull {
112+
if (it.file.name.startsWith("maven-metadata.xml")) {
113+
return@mapNotNull null
114+
}
115+
val newName = it.file.name.replaceBuildNumber(gav.artifactId, gav.version, buildNumber)
116+
FileWithPath(it.file, "$gavPath/$newName")
117+
}
118+
119+
transport.uploadFiles(renamedFiles)
120+
121+
val bytes = encodeToXml(newVersionMetadata).toByteArray()
122+
transport.put(versionMetadataPath, bytes)
154123
setOf("md5", "sha1", "sha256", "sha512").forEach {
155-
transport.put("$artifactMetadataPath.$it", bytes.digest(it.uppercase()))
124+
transport.put("$versionMetadataPath.$it", bytes.digest(it.uppercase()))
156125
}
126+
} else {
127+
/**
128+
* Not a snapshot, plainly update all the files
129+
*/
130+
transport.uploadFiles(gavFiles)
131+
}
132+
133+
/**
134+
* Update the [artifact metadata](https://maven.apache.org/repositories/metadata.html).
135+
*
136+
* See https://repo1.maven.org/maven2/com/apollographql/apollo/apollo-api-jvm/maven-metadata.xml for an example.
137+
*/
138+
val index = gavPath.lastIndexOf('/')
139+
check(index != -1) {
140+
"Nmcp: invalid gav path: '$gavPath'"
141+
}
142+
val artifactMetadataPath = "${gavPath.substring(0, index)}/maven-metadata.xml"
143+
val localArtifactMetadataFile = allFiles.firstOrNull { it.normalizedPath == artifactMetadataPath }
144+
val localArtifactMetadata = if (localArtifactMetadataFile == null) {
145+
// The publisher did not artifact level metadata, let's
146+
ArtifactMetadata(
147+
groupId = gav.groupId,
148+
artifactId = gav.artifactId,
149+
versioning = ArtifactMetadata.Versioning(
150+
latest = gav.version,
151+
release = gav.version,
152+
versions = emptyList(),
153+
lastUpdated = lastUpdated,
154+
),
155+
)
156+
} else {
157+
xml.decodeFromString<ArtifactMetadata>(localArtifactMetadataFile.file.readText())
158+
}
159+
160+
val remoteArtifactMetadata = transport.get(artifactMetadataPath)
157161

162+
val existingVersions = if (remoteArtifactMetadata != null) {
163+
xml.decodeFromString<ArtifactMetadata>(remoteArtifactMetadata.use { it.readUtf8() }).versioning.versions
164+
} else {
165+
emptyList()
166+
}
167+
168+
/**
169+
* See https://github.com/gradle/gradle/blob/cb0c615fb8e3690971bb7f89ad80f58943360624/platforms/software/maven/src/main/java/org/gradle/api/publish/maven/internal/publisher/AbstractMavenPublisher.java#L116.
170+
*/
171+
val versions = existingVersions.toMutableList()
172+
if (!versions.none { it == gav.version }) {
173+
versions.add(gav.version)
174+
}
175+
val newArtifactMetadata = localArtifactMetadata.copy(
176+
versioning = localArtifactMetadata.versioning.copy(
177+
versions = versions,
178+
),
179+
)
180+
181+
val bytes = encodeToXml(newArtifactMetadata).toByteArray()
182+
transport.put(artifactMetadataPath, bytes)
183+
setOf("md5", "sha1", "sha256", "sha512").forEach {
184+
transport.put("$artifactMetadataPath.$it", bytes.digest(it.uppercase()))
158185
}
159186
}
160187

@@ -168,7 +195,15 @@ private fun Transport.uploadFiles(filesWithPath: List<FileWithPath>) {
168195
internal fun timestampNow(): String {
169196
val now = Instant.now().atZone(java.time.ZoneOffset.UTC)
170197

171-
return String.format("%04d%02d%02d%02d%02d%02d", now.year, now.monthValue, now.dayOfMonth, now.hour, now.minute, now.second)
198+
return String.format(
199+
"%04d%02d%02d%02d%02d%02d",
200+
now.year,
201+
now.monthValue,
202+
now.dayOfMonth,
203+
now.hour,
204+
now.minute,
205+
now.second,
206+
)
172207
}
173208

174209
/**

0 commit comments

Comments
 (0)