Skip to content

Commit e1338fc

Browse files
damaru-incMichaelDavisSolaceasyncapi-bot
authored
fix: improve the handling of Avro schemas (#155)
* fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * Avro schemas are now named based on their name and namespace. * Removed file that wasn't supposed to be checked in. * fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * Avro schemas are now named based on their name and namespace. * Removed file that wasn't supposed to be checked in. * Fixed linter problems. * Add a technical requirements section to the README to note the required generator version. * feat: add a comment to methods listing which message types they use. (#163) Co-authored-by: Michael Davis <[email protected]> * chore(release): v0.11.0 * Updated package.json with the minimum version of generator. * fix: improve the handling of Avro schemas * Minor refactoring. * Fixed linter problem. * Avro schemas are now named based on their name and namespace. * Add a technical requirements section to the README to note the required generator version. * Updated package.json with the minimum version of generator. Co-authored-by: Michael Davis <[email protected]> Co-authored-by: asyncapi-bot <[email protected]>
1 parent 830a803 commit e1338fc

File tree

8 files changed

+269
-24
lines changed

8 files changed

+269
-24
lines changed

‎README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ The Spring Cloud Stream microservice generated using this template will be an _a
88

99
Note that this template ignores the 'Servers' section of AsyncAPI documents. The main reason for this is because SCSt does not directly work with messaging protocols. Protocols are implementation details specific to binders, and SCSt applications need not know or care which protocol is being used.
1010

11+
## Technical requirements
12+
13+
- 1.8.6 =< [Generator](https://github.com/asyncapi/generator/)
14+
- Generator specific [requirements](https://github.com/asyncapi/generator/#requirements)
15+
1116
## Specification Conformance
1217
Note that this template interprets the AsyncAPI document in conformance with the [AsyncAPI Specification](https://www.asyncapi.com/docs/specifications/2.0.0/).
1318
This means that when the template sees a subscribe operation, it will generate code to publish to that operation's channel.

‎hooks/post-process.js

Lines changed: 63 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
// vim: set ts=2 sw=2 sts=2 expandtab :
22
const fs = require('fs');
33
const path = require('path');
4-
const _ = require('lodash');
4+
const ScsLib = require('../lib/scsLib.js');
5+
const scsLib = new ScsLib();
56
// To enable debug logging, set the env var DEBUG="postProcess" with whatever things you want to see.
67
const debugPostProcess = require('debug')('postProcess');
78

@@ -14,14 +15,22 @@ module.exports = {
1415
const info = asyncapi.info();
1516
let javaPackage = generator.templateParams['javaPackage'];
1617
const extensions = info.extensions();
18+
let overridePath;
1719

1820
if (!javaPackage && info && extensions) {
1921
javaPackage = extensions['x-java-package'];
2022
}
2123

2224
if (javaPackage) {
2325
debugPostProcess(`package: ${javaPackage}`);
24-
const overridePath = `${generator.targetDir + sourceHead + javaPackage.replace(/\./g, '/')}/`;
26+
overridePath = `${generator.targetDir + sourceHead + javaPackage.replace(/\./g, '/')}/`;
27+
}
28+
29+
asyncapi.allSchemas().forEach((value, key, map) => {
30+
processSchema(key, value);
31+
});
32+
33+
if (javaPackage) {
2534
debugPostProcess(`Moving files from ${sourcePath} to ${overridePath}`);
2635
let first = true;
2736
fs.readdirSync(sourcePath).forEach(file => {
@@ -33,8 +42,7 @@ module.exports = {
3342
}
3443

3544
debugPostProcess(`Copying ${file}`);
36-
fs.copyFileSync(path.resolve(sourcePath, file), path.resolve(overridePath, file));
37-
fs.unlinkSync(path.resolve(sourcePath, file));
45+
moveFile(sourcePath, overridePath, file);
3846
}
3947
});
4048
sourcePath = overridePath;
@@ -54,29 +62,63 @@ module.exports = {
5462

5563
// This renames schema objects ensuring they're proper Java class names. It also removes files that are schemas of simple types.
5664

57-
const schemas = asyncapi.components().schemas();
58-
debugPostProcess('schemas:');
59-
debugPostProcess(schemas);
65+
function processSchema(schemaName, schema) {
66+
if (schemaName.startsWith('<')) {
67+
debugPostProcess(`found an anonymous schema ${schemaName}`);
68+
schemaName = schemaName.replace('<', '');
69+
schemaName = schemaName.replace('>', '');
70+
}
71+
72+
// First see if we need to move it to a different package based on its namespace.
73+
// This mainly applies to Avro files which have the fully qualified name.
74+
let newSourceDir = sourcePath;
75+
const generatedFileName = `${schemaName}.java`;
76+
let desiredClassName = scsLib.getClassName(schemaName);
6077

61-
for (const schemaName in asyncapi.components().schemas()) {
62-
const schema = schemas[schemaName];
63-
const type = schema.type();
64-
debugPostProcess(`postprocess schema ${schemaName} ${type}`);
65-
const oldPath = path.resolve(sourcePath, `${schemaName}.java`);
78+
const indexOfDot = schemaName.lastIndexOf('.');
79+
if (indexOfDot > 0) {
80+
const newPackage = schemaName.substring(0, indexOfDot);
81+
const className = schemaName.substring(indexOfDot + 1);
82+
debugPostProcess(`package: ${newPackage} className: ${className}`);
83+
newSourceDir = `${generator.targetDir + sourceHead + newPackage.replace(/\./g, '/')}/`;
84+
moveFile(sourcePath, newSourceDir, generatedFileName);
85+
desiredClassName = scsLib.getClassName(className);
86+
}
6687

67-
if (type === 'object' || type === 'enum') {
68-
let javaName = _.camelCase(schemaName);
69-
javaName = _.upperFirst(javaName);
88+
const oldPath = path.resolve(newSourceDir, generatedFileName);
89+
debugPostProcess(`old path: ${oldPath}`);
7090

71-
if (javaName !== schemaName) {
72-
const newPath = path.resolve(sourcePath, `${javaName}.java`);
73-
fs.renameSync(oldPath, newPath);
74-
debugPostProcess(`Renamed class file ${schemaName} to ${javaName}`);
91+
if (fs.existsSync(oldPath)) {
92+
const schemaType = schema.type();
93+
debugPostProcess(`Old path exists. schemaType: ${schemaType}`);
94+
if (schemaType === 'object' || schemaType === 'enum') {
95+
const javaName = scsLib.getClassName(schemaName);
96+
debugPostProcess(`desiredClassName: ${desiredClassName} schemaName: ${schemaName}`);
97+
98+
if (javaName !== schemaName) {
99+
const newPath = path.resolve(newSourceDir, `${desiredClassName}.java`);
100+
fs.renameSync(oldPath, newPath);
101+
debugPostProcess(`Renamed class file ${schemaName} to ${desiredClassName}`);
102+
}
103+
} else {
104+
// In this case it's an anonymous schema for a primitive type or something.
105+
debugPostProcess(`deleting ${oldPath}`);
106+
fs.unlinkSync(oldPath);
75107
}
76-
} else {
77-
fs.unlinkSync(oldPath);
78108
}
79109
}
110+
111+
function moveFile(oldDirectory, newDirectory, fileName) {
112+
if (!fs.existsSync(newDirectory)) {
113+
fs.mkdirSync(newDirectory, { recursive: true });
114+
debugPostProcess(`Made directory ${newDirectory}`);
115+
}
116+
const oldPath = path.resolve(oldDirectory, fileName);
117+
const newPath = path.resolve(newDirectory, fileName);
118+
fs.copyFileSync(oldPath, newPath);
119+
fs.unlinkSync(oldPath);
120+
debugPostProcess(`Moved ${fileName} from ${oldPath} to ${newPath}`);
121+
}
80122
}
81123
};
82124

‎lib/scsLib.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ const _ = require('lodash');
44
class ScsLib {
55
// This returns a valid Java class name.
66
getClassName(name) {
7-
const ret = this.getIdentifierName(name);
7+
const ret = _.camelCase(name);
88
return _.upperFirst(ret);
99
}
1010

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
]
6666
},
6767
"generator": {
68-
"generator": ">=0.50.0 <2.0.0",
68+
"generator": ">=1.8.6 <2.0.0",
6969
"parameters": {
7070
"actuator": {
7171
"description": "If present, it adds the dependencies for spring-boot-starter-web, spring-boot-starter-actuator and micrometer-registry-prometheus.",

‎template/src/main/java/$$schema$$.java renamed to ‎template/src/main/java/$$objectSchema$$.java

File renamed without changes.

‎test/__snapshots__/integration.test.js.snap

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,139 @@
11
// Jest Snapshot v1, https://goo.gl/fbAQLP
22

3+
exports[`template integration tests using the generator avro schemas should appear in a package based on their namespace, if any. 1`] = `
4+
"package com.acme;
5+
6+
import com.fasterxml.jackson.annotation.JsonInclude;
7+
8+
9+
@JsonInclude(JsonInclude.Include.NON_NULL)
10+
public class User {
11+
12+
public User () {
13+
}
14+
public User (
15+
String displayName,
16+
String email,
17+
Integer age) {
18+
this.displayName = displayName;
19+
this.email = email;
20+
this.age = age;
21+
}
22+
23+
24+
private String displayName;
25+
private String email;
26+
private Integer age;
27+
28+
public String getDisplayName() {
29+
return displayName;
30+
}
31+
32+
public User setDisplayName(String displayName) {
33+
this.displayName = displayName;
34+
return this;
35+
}
36+
37+
38+
public String getEmail() {
39+
return email;
40+
}
41+
42+
public User setEmail(String email) {
43+
this.email = email;
44+
return this;
45+
}
46+
47+
48+
public Integer getAge() {
49+
return age;
50+
}
51+
52+
public User setAge(Integer age) {
53+
this.age = age;
54+
return this;
55+
}
56+
57+
58+
public String toString() {
59+
return \\"User [\\"
60+
+ \\" displayName: \\" + displayName
61+
+ \\" email: \\" + email
62+
+ \\" age: \\" + age
63+
+ \\" ]\\";
64+
}
65+
}
66+
67+
"
68+
`;
69+
70+
exports[`template integration tests using the generator avro schemas should appear in a package based on their namespace, if any. 2`] = `
71+
"package com.acme;
72+
73+
import com.fasterxml.jackson.annotation.JsonInclude;
74+
75+
76+
@JsonInclude(JsonInclude.Include.NON_NULL)
77+
public class UserpublisherUser {
78+
79+
public UserpublisherUser () {
80+
}
81+
public UserpublisherUser (
82+
String displayName,
83+
String email,
84+
Integer age) {
85+
this.displayName = displayName;
86+
this.email = email;
87+
this.age = age;
88+
}
89+
90+
91+
private String displayName;
92+
private String email;
93+
private Integer age;
94+
95+
public String getDisplayName() {
96+
return displayName;
97+
}
98+
99+
public UserpublisherUser setDisplayName(String displayName) {
100+
this.displayName = displayName;
101+
return this;
102+
}
103+
104+
105+
public String getEmail() {
106+
return email;
107+
}
108+
109+
public UserpublisherUser setEmail(String email) {
110+
this.email = email;
111+
return this;
112+
}
113+
114+
115+
public Integer getAge() {
116+
return age;
117+
}
118+
119+
public UserpublisherUser setAge(Integer age) {
120+
this.age = age;
121+
return this;
122+
}
123+
124+
125+
public String toString() {
126+
return \\"UserpublisherUser [\\"
127+
+ \\" displayName: \\" + displayName
128+
+ \\" email: \\" + email
129+
+ \\" age: \\" + age
130+
+ \\" ]\\";
131+
}
132+
}
133+
134+
"
135+
`;
136+
3137
exports[`template integration tests using the generator should generate a comment for a consumer receiving multiple messages 1`] = `
4138
"
5139
import java.util.function.Consumer;

‎test/integration.test.js

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,4 +100,29 @@ describe('template integration tests using the generator', () => {
100100
expect(file).toMatchSnapshot();
101101
}
102102
});
103-
});
103+
104+
it('avro schemas should appear in a package based on their namespace, if any.', async () => {
105+
// Note that this file has 2 Avro schemas named User, but one has the namespace 'userpublisher.'
106+
const OUTPUT_DIR = generateFolderName();
107+
const PACKAGE = 'com.acme';
108+
const PACKAGE_PATH = path.join(...PACKAGE.split('.'));
109+
const AVRO_PACKAGE_PATH = 'userpublisher';
110+
const params = {
111+
binder: 'kafka',
112+
javaPackage: PACKAGE,
113+
artifactId: 'asyncApiFileName'
114+
};
115+
116+
const generator = new Generator(path.normalize('./'), OUTPUT_DIR, { forceWrite: true, templateParams: params });
117+
await generator.generateFromFile(path.resolve('test', 'mocks/kafka-avro.yaml'));
118+
119+
const expectedFiles = [
120+
`src/main/java/${PACKAGE_PATH}/User.java`,
121+
`src/main/java/${AVRO_PACKAGE_PATH}/User.java`,
122+
];
123+
for (const index in expectedFiles) {
124+
const file = await readFile(path.join(OUTPUT_DIR, expectedFiles[index]), 'utf8');
125+
expect(file).toMatchSnapshot();
126+
}
127+
});
128+
});

‎test/mocks/kafka-avro.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
asyncapi: '2.0.0'
2+
info:
3+
title: Avro Test
4+
version: '1.0.0'
5+
description: Tests Avro schema generation
6+
channels:
7+
userUpdates:
8+
publish:
9+
bindings:
10+
kafka:
11+
groupId: my-group
12+
message:
13+
schemaFormat: 'application/vnd.apache.avro;version=1.9.0'
14+
payload:
15+
name: User
16+
namespace: userpublisher
17+
type: record
18+
doc: User information
19+
fields:
20+
- name: displayName
21+
type: string
22+
- name: email
23+
type: string
24+
- name: age
25+
type: int
26+
subscribe:
27+
message:
28+
schemaFormat: 'application/vnd.apache.avro;version=1.9.0'
29+
payload:
30+
name: User
31+
type: record
32+
doc: User information
33+
fields:
34+
- name: displayName
35+
type: string
36+
- name: email
37+
type: string
38+
- name: age
39+
type: int

0 commit comments

Comments
 (0)