Skip to content

ipgallery.common.java / microservice

Go to a project
Commit fffb5b8a by Amir Aharon
Options

first working test of pulsar pub sub implementation

parent b09167c5
Expand all
Inline Side-by-side
Showing with 101 additions and 23 deletions
README.md
### Microservice Framework in JAVA ### Microservice Framework in JAVA
## 2.2.0
- Add Pulsar PubSub Implementation
- Add Either as default for optional
- Divide src to common and main so that app's can use only common where needed
## 2.1.1 ## 2.1.1
- downgrade metrics influxdb to fit the reporter on iot jar - downgrade metrics influxdb to fit the reporter on iot jar
......
build.gradle
group 'com.ipgallery.common' group 'com.ipgallery.common'
version '2.1.1' version '2.2.0'
apply plugin: 'java' apply plugin: 'java'
apply plugin: 'maven-publish' apply plugin: 'maven-publish'
......
src/common/java/common/microservice/context/PubSubMsg.java
...@@ -15,6 +15,12 @@ public class PubSubMsg { ...@@ -15,6 +15,12 @@ public class PubSubMsg {
this.mcid = mcid; this.mcid = mcid;
} }
public PubSubMsg(String content, String mcid, Map<String, String> parameters) {
this.content = content;
this.mcid = mcid;
this.parameters = parameters;
}
PubSubMsg addParameter(String param, String value){ PubSubMsg addParameter(String param, String value){
if (parameters == null) if (parameters == null)
parameters = new HashMap<>(); parameters = new HashMap<>();
...@@ -33,4 +39,9 @@ public class PubSubMsg { ...@@ -33,4 +39,9 @@ public class PubSubMsg {
public String getMcid() { public String getMcid() {
return mcid; return mcid;
} }
@Override
public String toString() {
return "mcid: " + mcid + ", content: " + content;
}
} }
src/common/java/common/microservice/services/CommonServices.java
...@@ -181,7 +181,7 @@ public class CommonServices { ...@@ -181,7 +181,7 @@ public class CommonServices {
} }
public static abstract class IPubSubService extends IService { public static abstract class IPubSubService extends IService {
public class PubSubMsgContext implements IMsgContext { public static class PubSubMsgContext implements IMsgContext {
private String topic; private String topic;
private String msg; private String msg;
private String mcid = null; private String mcid = null;
...@@ -227,6 +227,10 @@ public class CommonServices { ...@@ -227,6 +227,10 @@ public class CommonServices {
return this; return this;
} }
public Map<String, String> getParameters() {
return parameters;
}
@Override @Override
public void setParameters(Map<String, String> parameters) { public void setParameters(Map<String, String> parameters) {
this.parameters = parameters; this.parameters = parameters;
......
src/main/java/microservice/types/BaseRestResponse.java → src/common/java/common/microservice/types/BaseRestResponse.java
package microservice.types; package common.microservice.types;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
......
src/main/java/microservice/types/UserProfile.java → src/common/java/common/microservice/types/UserProfile.java
package microservice.types; package common.microservice.types;
import io.jsonwebtoken.Claims; import io.jsonwebtoken.Claims;
import io.jsonwebtoken.impl.DefaultClaims; import io.jsonwebtoken.impl.DefaultClaims;
import microservice.services.CommonServices; import common.microservice.services.CommonServices;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
......
src/main/java/microservice/utils/ServiceBuilderFactory.java
...@@ -330,6 +330,8 @@ public class ServiceBuilderFactory { ...@@ -330,6 +330,8 @@ public class ServiceBuilderFactory {
CommonServices.EnumPubSubServiceMode serviceMode = CommonServices.EnumPubSubServiceMode.E_BOTH; CommonServices.EnumPubSubServiceMode serviceMode = CommonServices.EnumPubSubServiceMode.E_BOTH;
int consumerPoolSize = 0; int consumerPoolSize = 0;
String serviceUrl = null; String serviceUrl = null;
String adminUrl = null;
String clusters = "standalone";
public PubSubServicePulsarBuilder(CommonServices.EnumPubSubServiceMode serviceMode) { public PubSubServicePulsarBuilder(CommonServices.EnumPubSubServiceMode serviceMode) {
this.serviceMode = serviceMode; this.serviceMode = serviceMode;
...@@ -345,6 +347,16 @@ public class ServiceBuilderFactory { ...@@ -345,6 +347,16 @@ public class ServiceBuilderFactory {
return this; return this;
} }
public PubSubServicePulsarBuilder setAdminUrl(String adminUrl) {
this.adminUrl = adminUrl;
return this;
}
public PubSubServicePulsarBuilder setClusters(String clusters) {
this.clusters = clusters;
return this;
}
private boolean validateParams() { private boolean validateParams() {
/** /**
* defaulting to number of processors * defaulting to number of processors
...@@ -352,7 +364,7 @@ public class ServiceBuilderFactory { ...@@ -352,7 +364,7 @@ public class ServiceBuilderFactory {
if (consumerPoolSize == 0){ if (consumerPoolSize == 0){
consumerPoolSize = Runtime.getRuntime().availableProcessors(); consumerPoolSize = Runtime.getRuntime().availableProcessors();
} }
if(serviceUrl == null) if(serviceUrl == null && adminUrl != null)
return false; return false;
return true; return true;
...@@ -362,7 +374,7 @@ public class ServiceBuilderFactory { ...@@ -362,7 +374,7 @@ public class ServiceBuilderFactory {
public CommonServices.IService build() { public CommonServices.IService build() {
if (validateParams()) { if (validateParams()) {
try { try {
pubSubServicePulsar = new IPubSubServicePulsarImpl(serviceUrl,consumerPoolSize); pubSubServicePulsar = new IPubSubServicePulsarImpl(serviceUrl,adminUrl,clusters,consumerPoolSize);
pubSubServicePulsar.setServiceMode(serviceMode); pubSubServicePulsar.setServiceMode(serviceMode);
} catch (Exception exp){ } catch (Exception exp){
......
src/test/java/microservice/TestMicroserviceApp.java
...@@ -103,6 +103,8 @@ public class TestMicroserviceApp { ...@@ -103,6 +103,8 @@ public class TestMicroserviceApp {
.build(); .build();
CommonServices.IService pulsarPubSub = ServiceBuilderFactory.createPubSubServicePulsarBuilder(CommonServices.EnumPubSubServiceMode.E_BOTH) CommonServices.IService pulsarPubSub = ServiceBuilderFactory.createPubSubServicePulsarBuilder(CommonServices.EnumPubSubServiceMode.E_BOTH)
.setServiceUrl("localhost:6650") .setServiceUrl("localhost:6650")
.setAdminUrl("localhost:8080")
.setClusters("standalone")
.build(); .build();
microservice.MicroserviceApp msApp = new microservice.MicroserviceApp(appName); microservice.MicroserviceApp msApp = new microservice.MicroserviceApp(appName);
...@@ -169,12 +171,28 @@ public class TestMicroserviceApp { ...@@ -169,12 +171,28 @@ public class TestMicroserviceApp {
(msgCtx,orgService) -> { (msgCtx,orgService) -> {
testZmqMsgQueue((CommonServices.IMsgQService.MsgQContext)msgCtx); testZmqMsgQueue((CommonServices.IMsgQService.MsgQContext)msgCtx);
})); }));
methodParamsList.add(new CommonServices.MethodParams(Enums.EnumServiceType.E_PUBSUB,
CommonServices.EnumPubSubCommands.E_NOTIFY,
"/activity",
(msgCtx,orgService) -> {
testPubSubMsg((CommonServices.IPubSubService.PubSubMsgContext)msgCtx);
}));
} }
long startTime = 0; long startTime = 0;
ObjectMapper objectMapper = new ObjectMapper(); ObjectMapper objectMapper = new ObjectMapper();
private void testPubSubMsg(CommonServices.IPubSubService.PubSubMsgContext msgCtx) {
// System.out.println("Recieved mcid: " + msgCtx.getMcid());
try {
JsonNode jsonNode = objectMapper.readValue(msgCtx.getMsg(), JsonNode.class);
handleJsonTestStartStop(jsonNode);
} catch (IOException e) {
e.printStackTrace();
}
}
/** /**
* around 300k/s * around 300k/s
* @param msgCtx * @param msgCtx
...@@ -182,6 +200,13 @@ public class TestMicroserviceApp { ...@@ -182,6 +200,13 @@ public class TestMicroserviceApp {
private void testZmqMsgQueue(CommonServices.IMsgQService.MsgQContext msgCtx) { private void testZmqMsgQueue(CommonServices.IMsgQService.MsgQContext msgCtx) {
try { try {
JsonNode jsonNode = objectMapper.readValue(msgCtx.msg, JsonNode.class); JsonNode jsonNode = objectMapper.readValue(msgCtx.msg, JsonNode.class);
handleJsonTestStartStop(jsonNode);
} catch (IOException e) {
e.printStackTrace();
}
}
private void handleJsonTestStartStop(JsonNode jsonNode) {
String state = jsonNode.path("state").asText(); String state = jsonNode.path("state").asText();
switch (state){ switch (state){
case "start": case "start":
...@@ -193,9 +218,6 @@ public class TestMicroserviceApp { ...@@ -193,9 +218,6 @@ public class TestMicroserviceApp {
case "msg": case "msg":
break; break;
} }
} catch (IOException e) {
e.printStackTrace();
}
} }
private void testZmqRead(RestContext msgCtx) { private void testZmqRead(RestContext msgCtx) {
......
src/test/java/microservice/TestPulsar.java
package microservice; package microservice;
import common.microservice.context.PubSubMsg;
import common.microservice.utils.IDGenerator; import common.microservice.utils.IDGenerator;
import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminBuilder;
...@@ -9,6 +10,7 @@ import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl; ...@@ -9,6 +10,7 @@ import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl;
import org.apache.pulsar.client.api.*; import org.apache.pulsar.client.api.*;
import org.apache.pulsar.client.api.url.URL; import org.apache.pulsar.client.api.url.URL;
import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
import org.apache.pulsar.client.impl.schema.JSONSchema;
import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfo;
import org.junit.Test; import org.junit.Test;
...@@ -28,18 +30,20 @@ public class TestPulsar { ...@@ -28,18 +30,20 @@ public class TestPulsar {
public void testTopicProducer() throws PulsarClientException { public void testTopicProducer() throws PulsarClientException {
String localClusterUrl = "pulsar://localhost:6650"; String localClusterUrl = "pulsar://localhost:6650";
String namespace = "mcx/testApp"; // This namespace is created automatically String namespace = "mcx/testApp"; // This namespace is created automatically
String topic = String.format("persistent://%s/my-topic1", namespace); String topic = String.format("persistent://%s/activity", namespace);
PulsarClient client = PulsarClient.builder() PulsarClient client = PulsarClient.builder()
.serviceUrl(localClusterUrl) .serviceUrl(localClusterUrl)
.build(); .build();
Producer producer = client.newProducer().topic(topic).create(); Producer<PubSubMsg> producer = client.newProducer(JSONSchema.of(PubSubMsg.class))
.topic(topic)
.create();
// Publish 10 messages to the topic // Publish 10 messages to the topic
for (int i = 0; i < 10; i++) { for (int i = 0; i < 100; i++) {
producer.send(String.format("Message number %d", i).getBytes()); PubSubMsg pubSubMsg = new PubSubMsg("hello - " + String.valueOf(System.currentTimeMillis()),String.valueOf(i));
producer.send(pubSubMsg);
System.out.println("Sending message"); System.out.println("Sending message");
// final Message message = MessageBuilder.create() // final Message message = MessageBuilder.create()
// .setContent(String.format("Message number %d", i).getBytes()) // .setContent(String.format("Message number %d", i).getBytes())
...@@ -65,17 +69,24 @@ public class TestPulsar { ...@@ -65,17 +69,24 @@ public class TestPulsar {
//final Consumer consumer = client.newConsumer().subscriptionName("my-sub-1").topic(topic).subscribe(); //final Consumer consumer = client.newConsumer().subscriptionName("my-sub-1").topic(topic).subscribe();
Pattern allTopicsInNamespace = Pattern.compile("persistent://mcx/testApp/.*"); Pattern allTopicsInNamespace = Pattern.compile("persistent://mcx/testApp/.*");
final Consumer consumer = client.newConsumer() final Consumer<PubSubMsg> consumer = client.newConsumer(JSONSchema.of(PubSubMsg.class))
.subscriptionName(IDGenerator.createUUID()) .subscriptionName(IDGenerator.createUUID())
//.subscriptionType(SubscriptionType.Shared) // enable for multi-instance .subscriptionType(SubscriptionType.Shared) // enable for multi-instance
.topicsPattern(allTopicsInNamespace) .topicsPattern(allTopicsInNamespace)
.subscribe(); .subscribe();
// final Consumer consumer = client.newConsumer()
// .subscriptionName(IDGenerator.createUUID())
// .subscriptionType(SubscriptionType.Shared) // enable for multi-instance
// .topicsPattern(allTopicsInNamespace)
// .subscribe();
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
// Wait for a message // Wait for a message
Message msg = consumer.receive(); Message<PubSubMsg> msg = consumer.receive();
System.out.printf("Message received: %s\n", new String(msg.getData())); System.out.printf("Message received: %s\n", msg.getValue().toString());
// Acknowledge the message so that it can be deleted by the message broker // Acknowledge the message so that it can be deleted by the message broker
consumer.acknowledge(msg); consumer.acknowledge(msg);
......
src/test/java/microservice/TestServicesAndMethods.java
...@@ -36,7 +36,7 @@ import java.util.stream.Collectors; ...@@ -36,7 +36,7 @@ import java.util.stream.Collectors;
*/ */
public class TestServicesAndMethods { public class TestServicesAndMethods {
public static final int ITERATIONS = 10000000; public static final int ITERATIONS = 10000;
private static class RoutingMatch { private static class RoutingMatch {
...@@ -278,12 +278,25 @@ public class TestServicesAndMethods { ...@@ -278,12 +278,25 @@ public class TestServicesAndMethods {
@Test @Test
public void testPubSubPulsar() throws InterruptedException { public void testPubSubPulsar() throws InterruptedException {
CommonServices.IService iService = ServiceBuilderFactory.createPubSubServicePulsarBuilder(CommonServices.EnumPubSubServiceMode.E_BOTH) CommonServices.IService iService = ServiceBuilderFactory.createPubSubServicePulsarBuilder(CommonServices.EnumPubSubServiceMode.E_PUBLISHER)
.setServiceUrl("localhost:6650") .setServiceUrl("localhost:6650")
.setAdminUrl("localhost:8080")
.build(); .build();
CommonServices.IPubSubService pubSubService = (CommonServices.IPubSubService)iService; CommonServices.IPubSubService pubSubService = (CommonServices.IPubSubService)iService;
// pubSubService.init(); pubSubService.init();
// pubSubService.run(); pubSubService.run();
ObjectNode objectNode = JsonNodeFactory.instance.objectNode().put("state", "start").put("iterations", ITERATIONS);
System.out.println("Testing " + String.valueOf(ITERATIONS) + " iterations");
long start = System.currentTimeMillis();
pubSubService.publish(new CommonServices.IPubSubService.PubSubMsgContext("/activity", objectNode.toString()));
objectNode.put("state", "msg");
for (int i = 0; i < TestServicesAndMethods.ITERATIONS; i++) {
objectNode.put("msg","hello" + String.valueOf(i));
pubSubService.publish(new CommonServices.IPubSubService.PubSubMsgContext("/activity", objectNode.toString()));
}
objectNode.put("state", "end");
pubSubService.publish(new CommonServices.IPubSubService.PubSubMsgContext("/activity", objectNode.toString()));
System.out.println("Async publish Test of: " + String.valueOf(ITERATIONS) +" took (msec): " + String.valueOf(System.currentTimeMillis() - start));
Thread.sleep(1000); Thread.sleep(1000);
pubSubService.shutdown(); pubSubService.shutdown();
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment