Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
ipgallery.common.java
/
microservice
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Registry
Issues
0
Merge Requests
0
Pipelines
Wiki
Settings
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit
fffb5b8a
authored
Feb 21, 2019
by
Amir Aharon
Browse files
Options
_('Browse Files')
Download
Email Patches
Plain Diff
first working test of pulsar pub sub implementation
parent
b09167c5
Show whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
201 additions
and
54 deletions
README.md
build.gradle
src/common/java/common/microservice/context/PubSubMsg.java
src/common/java/common/microservice/services/CommonServices.java
src/main/java/microservice/types/BaseRestResponse.java → src/common/java/common/microservice/types/BaseRestResponse.java
src/main/java/microservice/types/UserProfile.java → src/common/java/common/microservice/types/UserProfile.java
src/main/java/microservice/services/IPubSubServicePulsarImpl.java
src/main/java/microservice/utils/ServiceBuilderFactory.java
src/test/java/microservice/TestMicroserviceApp.java
src/test/java/microservice/TestPulsar.java
src/test/java/microservice/TestServicesAndMethods.java
README.md
View file @
fffb5b8a
### Microservice Framework in JAVA
### Microservice Framework in JAVA
## 2.2.0
-
Add Pulsar PubSub Implementation
-
Add Either as default for optional
-
Divide src to common and main so that app's can use only common where needed
## 2.1.1
## 2.1.1
-
downgrade metrics influxdb to fit the reporter on iot jar
-
downgrade metrics influxdb to fit the reporter on iot jar
...
...
build.gradle
View file @
fffb5b8a
group
'com.ipgallery.common'
group
'com.ipgallery.common'
version
'2.
1.1
'
version
'2.
2.0
'
apply
plugin:
'java'
apply
plugin:
'java'
apply
plugin:
'maven-publish'
apply
plugin:
'maven-publish'
...
...
src/common/java/common/microservice/context/PubSubMsg.java
View file @
fffb5b8a
...
@@ -15,6 +15,12 @@ public class PubSubMsg {
...
@@ -15,6 +15,12 @@ public class PubSubMsg {
this
.
mcid
=
mcid
;
this
.
mcid
=
mcid
;
}
}
public
PubSubMsg
(
String
content
,
String
mcid
,
Map
<
String
,
String
>
parameters
)
{
this
.
content
=
content
;
this
.
mcid
=
mcid
;
this
.
parameters
=
parameters
;
}
PubSubMsg
addParameter
(
String
param
,
String
value
){
PubSubMsg
addParameter
(
String
param
,
String
value
){
if
(
parameters
==
null
)
if
(
parameters
==
null
)
parameters
=
new
HashMap
<>();
parameters
=
new
HashMap
<>();
...
@@ -33,4 +39,9 @@ public class PubSubMsg {
...
@@ -33,4 +39,9 @@ public class PubSubMsg {
public
String
getMcid
()
{
public
String
getMcid
()
{
return
mcid
;
return
mcid
;
}
}
@Override
public
String
toString
()
{
return
"mcid: "
+
mcid
+
", content: "
+
content
;
}
}
}
src/common/java/common/microservice/services/CommonServices.java
View file @
fffb5b8a
...
@@ -181,7 +181,7 @@ public class CommonServices {
...
@@ -181,7 +181,7 @@ public class CommonServices {
}
}
public
static
abstract
class
IPubSubService
extends
IService
{
public
static
abstract
class
IPubSubService
extends
IService
{
public
class
PubSubMsgContext
implements
IMsgContext
{
public
static
class
PubSubMsgContext
implements
IMsgContext
{
private
String
topic
;
private
String
topic
;
private
String
msg
;
private
String
msg
;
private
String
mcid
=
null
;
private
String
mcid
=
null
;
...
@@ -227,6 +227,10 @@ public class CommonServices {
...
@@ -227,6 +227,10 @@ public class CommonServices {
return
this
;
return
this
;
}
}
public
Map
<
String
,
String
>
getParameters
()
{
return
parameters
;
}
@Override
@Override
public
void
setParameters
(
Map
<
String
,
String
>
parameters
)
{
public
void
setParameters
(
Map
<
String
,
String
>
parameters
)
{
this
.
parameters
=
parameters
;
this
.
parameters
=
parameters
;
...
...
src/
main/java
/microservice/types/BaseRestResponse.java
→
src/
common/java/common
/microservice/types/BaseRestResponse.java
View file @
fffb5b8a
package
microservice
.
types
;
package
common
.
microservice
.
types
;
import
com.fasterxml.jackson.annotation.JsonCreator
;
import
com.fasterxml.jackson.annotation.JsonCreator
;
import
com.fasterxml.jackson.annotation.JsonIgnore
;
import
com.fasterxml.jackson.annotation.JsonIgnore
;
...
...
src/
main/java
/microservice/types/UserProfile.java
→
src/
common/java/common
/microservice/types/UserProfile.java
View file @
fffb5b8a
package
microservice
.
types
;
package
common
.
microservice
.
types
;
import
io.jsonwebtoken.Claims
;
import
io.jsonwebtoken.Claims
;
import
io.jsonwebtoken.impl.DefaultClaims
;
import
io.jsonwebtoken.impl.DefaultClaims
;
import
microservice.services.CommonServices
;
import
common.
microservice.services.CommonServices
;
import
java.util.HashMap
;
import
java.util.HashMap
;
import
java.util.Map
;
import
java.util.Map
;
...
...
src/main/java/microservice/services/IPubSubServicePulsarImpl.java
View file @
fffb5b8a
package
microservice
.
services
;
package
microservice
.
services
;
import
com.fasterxml.jackson.databind.JsonNode
;
import
common.JsonHandler
;
import
common.microservice.context.PubSubMsg
;
import
common.microservice.context.PubSubMsg
;
import
common.microservice.defs.Constants
;
import
common.microservice.defs.Constants
;
import
common.microservice.defs.Enums
;
import
common.microservice.defs.Enums
;
...
@@ -12,9 +9,19 @@ import common.microservice.services.CommonServices;
...
@@ -12,9 +9,19 @@ import common.microservice.services.CommonServices;
import
common.microservice.utils.IDGenerator
;
import
common.microservice.utils.IDGenerator
;
import
microservice.MicroserviceApp
;
import
microservice.MicroserviceApp
;
import
microservice.handlers.Reactor
;
import
microservice.handlers.Reactor
;
import
org.apache.pulsar.client.admin.PulsarAdmin
;
import
org.apache.pulsar.client.admin.PulsarAdminException
;
import
org.apache.pulsar.client.admin.Tenants
;
import
org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl
;
import
org.apache.pulsar.client.api.*
;
import
org.apache.pulsar.client.api.*
;
import
org.apache.pulsar.client.impl.schema.JSONSchema
;
import
org.apache.pulsar.common.policies.data.TenantInfo
;
import
java.util.Arrays
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Set
;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.ExecutorService
;
import
java.util.concurrent.ExecutorService
;
import
java.util.concurrent.Executors
;
import
java.util.concurrent.Executors
;
...
@@ -37,33 +44,48 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -37,33 +44,48 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
public
static
final
String
PERSISTENT_PREFIX
=
"persistent://"
;
public
static
final
String
PERSISTENT_PREFIX
=
"persistent://"
;
public
static
final
String
TOPIC_PREFIX
=
PERSISTENT_PREFIX
+
Constants
.
DEFAULT_TENANT
;
public
static
final
String
TOPIC_PREFIX
=
PERSISTENT_PREFIX
+
Constants
.
DEFAULT_TENANT
;
public
static
int
INIITIAL_PRODUCERS_SIZE
=
5
;
public
static
int
INIITIAL_PRODUCERS_SIZE
=
5
;
private
String
adminUrl
;
private
PulsarClient
client
=
null
;
private
PulsarClient
client
=
null
;
private
String
serviceUrl
;
private
String
serviceUrl
;
private
volatile
Consumer
<
byte
[]
>
consumer
=
null
;
private
Consumer
<
PubSubMsg
>
consumer
=
null
;
private
ILogger
logger
=
null
;
private
ILogger
logger
=
null
;
private
String
clientId
;
private
String
clientId
;
private
Thread
consumerThread
=
null
;
private
Thread
consumerThread
=
null
;
private
int
consumersThreadPoolSize
;
private
int
consumersThreadPoolSize
;
private
ExecutorService
consumerExecutorService
=
null
;
private
ExecutorService
consumerExecutorService
=
null
;
private
ConcurrentHashMap
<
String
,
Producer
<
byte
[]
>>
producersMap
=
new
ConcurrentHashMap
<>(
INIITIAL_PRODUCERS_SIZE
);
private
ConcurrentHashMap
<
String
,
Producer
<
PubSubMsg
>>
producersMap
=
new
ConcurrentHashMap
<>(
INIITIAL_PRODUCERS_SIZE
);
private
boolean
consumerRunning
=
false
;
private
boolean
consumerRunning
=
false
;
private
Set
<
String
>
clusters
;
// = new HashSet<>(Arrays.asList("standalone"));
private
String
namespacePrefix
;
public
IPubSubServicePulsarImpl
(
String
serviceUrl
,
public
IPubSubServicePulsarImpl
(
String
serviceUrl
,
String
adminUrl
,
String
clusters
,
int
consumersThreadPoolSize
)
{
int
consumersThreadPoolSize
)
{
if
(!
serviceUrl
.
startsWith
(
PULSAR_PREFIX
))
if
(!
serviceUrl
.
startsWith
(
PULSAR_PREFIX
))
serviceUrl
=
PULSAR_PREFIX
+
serviceUrl
;
serviceUrl
=
PULSAR_PREFIX
+
serviceUrl
;
this
.
serviceUrl
=
serviceUrl
;
this
.
serviceUrl
=
serviceUrl
;
this
.
adminUrl
=
adminUrl
;
this
.
clusters
=
new
HashSet
<>(
Arrays
.
asList
(
clusters
.
split
(
","
)));
this
.
consumersThreadPoolSize
=
consumersThreadPoolSize
;
this
.
consumersThreadPoolSize
=
consumersThreadPoolSize
;
}
}
@Override
@Override
public
boolean
init
()
{
public
boolean
init
()
{
final
MicroserviceApp
microserviceApp
=
MicroserviceApp
.
getsInstance
();
final
MicroserviceApp
microserviceApp
=
MicroserviceApp
.
getsInstance
();
if
(
microserviceApp
!=
null
)
{
clientId
=
microserviceApp
.
getAppName
()
+
'-'
clientId
=
microserviceApp
.
getAppName
()
+
'-'
+
microserviceApp
.
getServerName
()
+
'-'
+
microserviceApp
.
getServerName
()
+
'-'
+
microserviceApp
.
getId
();
+
microserviceApp
.
getId
();
logger
=
MicroserviceApp
.
getsInstance
().
getLogger
();
logger
=
MicroserviceApp
.
getsInstance
().
getLogger
();
namespacePrefix
=
TOPIC_PREFIX
+
"/"
+
microserviceApp
.
getAppName
();
}
else
{
// for tests where there is no MicroserviceApp
namespacePrefix
=
TOPIC_PREFIX
+
"/"
+
"testApp"
;
}
try
{
try
{
client
=
PulsarClient
.
builder
()
client
=
PulsarClient
.
builder
()
.
serviceUrl
(
this
.
serviceUrl
)
.
serviceUrl
(
this
.
serviceUrl
)
...
@@ -77,6 +99,9 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -77,6 +99,9 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
}
catch
(
PulsarClientException
e
)
{
}
catch
(
PulsarClientException
e
)
{
System
.
err
.
println
(
e
.
toString
());
System
.
err
.
println
(
e
.
toString
());
return
false
;
return
false
;
}
catch
(
PulsarAdminException
e
)
{
System
.
err
.
println
(
e
.
toString
());
return
false
;
}
}
return
true
;
return
true
;
}
}
...
@@ -86,40 +111,55 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -86,40 +111,55 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
if
(
getServiceMode
()
==
CommonServices
.
EnumPubSubServiceMode
.
E_PUBLISHER
)
if
(
getServiceMode
()
==
CommonServices
.
EnumPubSubServiceMode
.
E_PUBLISHER
)
return
;
return
;
consumerThread
=
new
Thread
(()
->
{
consumerThread
=
new
Thread
(()
->
{
// wait for message and dispatch it
// wait for message and dispatch it
consumerRunning
=
true
;
consumerRunning
=
true
;
logger
.
info
(
"Pulsar PubSub started successfully"
);
logger
.
info
(
"Pulsar PubSub started successfully"
);
try
{
try
{
while
(
consumerRunning
)
{
while
(
consumerRunning
)
{
Message
msg
=
consumer
.
receive
();
Message
<
PubSubMsg
>
msg
=
consumer
.
receive
();
try
{
final
PubSubMsgContext
msgCtx
=
getMsgContext
(
msg
);
final
PubSubMsgContext
msgCtx
=
getMsgContext
(
msg
);
if
(
msgCtx
!=
null
)
{
if
(
msgCtx
!=
null
)
{
logger
.
debug
(
SERVICE_NAME
+
" > Message received: "
+
msgCtx
.
toString
());
logger
.
debug
(
SERVICE_NAME
+
" > Message received: "
+
msgCtx
.
toString
());
consumerExecutorService
.
execute
(()
->
dispatchMsgCtx
(
msgCtx
));
// consumerExecutorService.execute(() -> dispatchMsgCtx(msgCtx));
dispatchMsgCtx
(
msgCtx
);
}
else
{
}
else
{
logger
.
warning
(
SERVICE_NAME
+
" > Failed to parse Pulsar message: "
+
msg
.
getMessageId
());
logger
.
warning
(
SERVICE_NAME
+
" > Failed to parse Pulsar message: "
+
msg
.
getMessageId
());
}
}
}
finally
{
// Acknowledge the message so that it can be deleted by the message broker
// Acknowledge the message so that it can be deleted by the message broker
consumer
.
acknowledge
(
msg
);
consumer
.
acknowledge
(
msg
);
}
}
}
}
catch
(
PulsarClientException
e
)
{
}
catch
(
PulsarClientException
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
}
}
});
});
consumerThread
.
setName
(
getClass
().
getName
()
+
"::PulsarConsumerThread"
);
consumerThread
.
start
();
}
}
/**
/**
* getting the actual topic and
* delegate to reactor
* delegate to reactor
* @param msgCtx
* @param msgCtx
*/
*/
private
void
dispatchMsgCtx
(
PubSubMsgContext
msgCtx
)
{
private
void
dispatchMsgCtx
(
PubSubMsgContext
msgCtx
)
{
String
topic
=
msgCtx
.
getTopic
();
//.substring(TOPIC_PREFIX.length());
if
(
topic
.
length
()
>
namespacePrefix
.
length
()
&&
topic
.
startsWith
(
namespacePrefix
))
{
topic
=
topic
.
substring
(
namespacePrefix
.
length
());
String
key
=
Reactor
.
buildServiceKey
(
Enums
.
EnumServiceType
.
E_PUBSUB
,
String
key
=
Reactor
.
buildServiceKey
(
Enums
.
EnumServiceType
.
E_PUBSUB
,
CommonServices
.
EnumPubSubCommands
.
E_NOTIFY
,
CommonServices
.
EnumPubSubCommands
.
E_NOTIFY
,
msgCtx
.
getTopic
());
topic
);
reactor
.
delegate
(
this
,
key
,
msgCtx
);
reactor
.
delegate
(
this
,
key
,
msgCtx
);
}
else
{
logger
.
warning
(
SERVICE_NAME
+
" > dispatchMsgCtx received topic with invalid/no namespace"
);
}
}
}
/**
/**
...
@@ -127,11 +167,9 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -127,11 +167,9 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
* @param msg
* @param msg
* @return
* @return
*/
*/
private
PubSubMsgContext
getMsgContext
(
Message
msg
)
{
private
PubSubMsgContext
getMsgContext
(
Message
<
PubSubMsg
>
msg
)
{
final
JsonNode
jsonNode
=
JsonHandler
.
getJsonNodeFromString
(
new
String
(
msg
.
getData
()));
final
PubSubMsg
pubSubMsg
=
msg
.
getValue
();
if
(
jsonNode
!=
null
){
PubSubMsg
pubSubMsg
=
(
PubSubMsg
)
JsonHandler
.
getNodeAsObject
(
jsonNode
,
PubSubMsg
.
class
);
if
(
pubSubMsg
!=
null
){
if
(
pubSubMsg
!=
null
){
return
new
PubSubMsgContext
(
msg
.
getTopicName
(),
return
new
PubSubMsgContext
(
msg
.
getTopicName
(),
pubSubMsg
.
getContent
(),
pubSubMsg
.
getContent
(),
...
@@ -140,12 +178,8 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -140,12 +178,8 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
}
else
{
}
else
{
logger
.
error
(
SERVICE_NAME
+
" > getMsgContext > Failed in parsing json to PubSubMsg"
);
logger
.
error
(
SERVICE_NAME
+
" > getMsgContext > Failed in parsing json to PubSubMsg"
);
}
}
}
else
{
logger
.
error
(
SERVICE_NAME
+
" > getMsgContext > Failed in parsing msg to json"
);
}
return
null
;
return
null
;
}
}
@Override
@Override
...
@@ -220,18 +254,21 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -220,18 +254,21 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
try
{
try
{
// create mcid if missing
// create mcid if missing
if
(
pubSubMsgContext
.
getMcid
()
==
null
&&
pubSubMsgContext
.
getMcid
().
isEmpty
())
{
if
(
pubSubMsgContext
.
getMcid
()
==
null
||
pubSubMsgContext
.
getMcid
().
isEmpty
())
{
pubSubMsgContext
.
setMcid
(
IDGenerator
.
createUUID
());
pubSubMsgContext
.
setMcid
(
IDGenerator
.
createUUID
());
final
Producer
<
byte
[]>
producerForTopic
=
getProducerForTopic
(
pubSubMsgContext
.
getTopic
());
final
Producer
<
PubSubMsg
>
producerForTopic
=
getProducerForTopic
(
pubSubMsgContext
.
getTopic
());
if
(
producerForTopic
!=
null
)
if
(
producerForTopic
!=
null
)
{
producerForTopic
.
sendAsync
(
pubSubMsgContext
.
getMsg
().
getBytes
()).
whenCompleteAsync
((
messageId
,
throwable
)
->
{
// construct msg to send
if
(
throwable
!=
null
){
PubSubMsg
pubSubMsg
=
new
PubSubMsg
(
pubSubMsgContext
.
getMsg
(),
pubSubMsgContext
.
getMcid
(),
pubSubMsgContext
.
getParameters
());
producerForTopic
.
sendAsync
(
pubSubMsg
).
whenCompleteAsync
((
messageId
,
throwable
)
->
{
if
(
throwable
!=
null
)
{
logger
.
error
(
SERVICE_NAME
+
" > Failed in sendAsync: "
+
throwable
.
getMessage
());
logger
.
error
(
SERVICE_NAME
+
" > Failed in sendAsync: "
+
throwable
.
getMessage
());
}
else
if
(
messageId
!=
null
)
{
}
else
if
(
messageId
!=
null
)
{
logger
.
debug
(
SERVICE_NAME
+
" > publish > sent msg-id: "
+
messageId
);
logger
.
debug
(
SERVICE_NAME
+
" > publish > sent msg-id: "
+
messageId
);
}
}
});
});
}
}
}
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
logger
.
error
(
SERVICE_NAME
+
" > publish > Failed to create producer/send msg: "
+
e
);
logger
.
error
(
SERVICE_NAME
+
" > publish > Failed to create producer/send msg: "
+
e
);
}
}
...
@@ -242,10 +279,14 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -242,10 +279,14 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
}
}
private
Producer
<
byte
[]
>
getProducerForTopic
(
String
topic
)
{
private
Producer
<
PubSubMsg
>
getProducerForTopic
(
String
topic
)
{
return
producersMap
.
computeIfAbsent
(
topic
,
t
->
{
return
producersMap
.
computeIfAbsent
(
topic
,
t
->
{
try
{
try
{
return
client
.
newProducer
().
topic
(
topic
).
create
();
Producer
<
PubSubMsg
>
producer
=
client
.
newProducer
(
JSONSchema
.
of
(
PubSubMsg
.
class
))
.
topic
(
topic
)
.
create
();
return
producer
;
//return client.newProducer().topic(topic).create();
}
catch
(
PulsarClientException
e
)
{
}
catch
(
PulsarClientException
e
)
{
logger
.
error
(
SERVICE_NAME
+
" > Failed to create producer for topic "
+
topic
);
logger
.
error
(
SERVICE_NAME
+
" > Failed to create producer for topic "
+
topic
);
}
}
...
@@ -258,8 +299,8 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -258,8 +299,8 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
if
(
topic
!=
null
&&
!
topic
.
isEmpty
()
&&
if
(
topic
!=
null
&&
!
topic
.
isEmpty
()
&&
pubSubMsgContext
.
getMsg
()
!=
null
&&
!
pubSubMsgContext
.
getMsg
().
isEmpty
()){
pubSubMsgContext
.
getMsg
()
!=
null
&&
!
pubSubMsgContext
.
getMsg
().
isEmpty
()){
// adding the topic prefix
// adding the topic prefix
if
(!
topic
.
startsWith
(
TOPIC_PREFIX
)){
if
(!
topic
.
startsWith
(
namespacePrefix
)){
topic
=
topic
.
charAt
(
0
)
==
'/'
?
TOPIC_PREFIX
+
topic
:
TOPIC_PREFIX
+
'/'
+
topic
;
topic
=
topic
.
charAt
(
0
)
==
'/'
?
namespacePrefix
+
topic
:
namespacePrefix
+
'/'
+
topic
;
pubSubMsgContext
.
setTopic
(
topic
);
pubSubMsgContext
.
setTopic
(
topic
);
}
}
return
true
;
return
true
;
...
@@ -267,19 +308,47 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
...
@@ -267,19 +308,47 @@ public class IPubSubServicePulsarImpl extends CommonServices.IPubSubService {
return
false
;
return
false
;
}
}
private
void
createConsumer
()
throws
PulsarClientException
{
private
void
createConsumer
()
throws
PulsarClientException
,
PulsarAdminException
{
/**
/**
* wildcard topic:
* wildcard topic:
* {persistent|non-persistent}://<tenant>/<namespace>/<regular expression>
* {persistent|non-persistent}://<tenant>/<namespace>/<regular expression>
*/
*/
final
MicroserviceApp
microserviceApp
=
MicroserviceApp
.
getsInstance
();
final
MicroserviceApp
microserviceApp
=
MicroserviceApp
.
getsInstance
();
String
pattern
=
TOPIC_PREFIX
+
"/"
+
microserviceApp
.
getAppName
()
+
"/.*"
;
String
pattern
=
namespacePrefix
+
"/.*"
;
adminValidateTenantAndNamespace
(
Constants
.
DEFAULT_TENANT
,
microserviceApp
.
getAppName
());
Pattern
allTopicsInNamespace
=
Pattern
.
compile
(
pattern
);
Pattern
allTopicsInNamespace
=
Pattern
.
compile
(
pattern
);
consumer
=
client
.
newConsumer
()
consumer
=
client
.
newConsumer
(
JSONSchema
.
of
(
PubSubMsg
.
class
)
)
.
subscriptionName
(
clientId
)
.
subscriptionName
(
clientId
)
.
subscriptionType
(
SubscriptionType
.
Shared
)
// enable for multi-instance
.
subscriptionType
(
SubscriptionType
.
Shared
)
// enable for multi-instance
.
topicsPattern
(
allTopicsInNamespace
)
.
topicsPattern
(
allTopicsInNamespace
)
.
subscribe
();
.
subscribe
();
consumerExecutorService
=
Executors
.
newFixedThreadPool
(
consumersThreadPoolSize
);
consumerExecutorService
=
Executors
.
newFixedThreadPool
(
consumersThreadPoolSize
);
}
}
private
void
adminValidateTenantAndNamespace
(
String
tenant
,
String
namespace
)
throws
PulsarAdminException
,
PulsarClientException
{
if
(!
adminUrl
.
startsWith
(
"http"
)){
adminUrl
=
"http://"
+
adminUrl
;
}
PulsarAdmin
admin
=
new
PulsarAdminBuilderImpl
().
serviceHttpUrl
(
adminUrl
).
build
();
final
Tenants
tenants
=
admin
.
tenants
();
final
List
<
String
>
tenantList
=
tenants
.
getTenants
();
final
TenantInfo
tenantInfo
=
new
TenantInfo
();
tenantInfo
.
setAllowedClusters
(
clusters
);
if
(!
tenantList
.
contains
(
tenant
)){
// creating tenant
logger
.
info
(
SERVICE_NAME
+
" >> Creating tenant: "
+
tenant
);
tenants
.
createTenant
(
tenant
,
tenantInfo
);
}
final
List
<
String
>
namespaces
=
admin
.
namespaces
().
getNamespaces
(
tenant
);
String
namespaceStr
=
tenant
+
'/'
+
namespace
;
if
(!
namespaces
.
contains
(
namespaceStr
)){
logger
.
info
(
SERVICE_NAME
+
" >> Creating namespace: "
+
namespaceStr
);
admin
.
namespaces
().
createNamespace
(
namespaceStr
);
admin
.
namespaces
().
setNamespaceMessageTTL
(
namespaceStr
,
3600
);
admin
.
namespaces
().
setNamespaceReplicationClusters
(
namespaceStr
,
clusters
);
}
admin
.
close
();
}
}
}
src/main/java/microservice/utils/ServiceBuilderFactory.java
View file @
fffb5b8a
...
@@ -330,6 +330,8 @@ public class ServiceBuilderFactory {
...
@@ -330,6 +330,8 @@ public class ServiceBuilderFactory {
CommonServices
.
EnumPubSubServiceMode
serviceMode
=
CommonServices
.
EnumPubSubServiceMode
.
E_BOTH
;
CommonServices
.
EnumPubSubServiceMode
serviceMode
=
CommonServices
.
EnumPubSubServiceMode
.
E_BOTH
;
int
consumerPoolSize
=
0
;
int
consumerPoolSize
=
0
;
String
serviceUrl
=
null
;
String
serviceUrl
=
null
;
String
adminUrl
=
null
;
String
clusters
=
"standalone"
;
public
PubSubServicePulsarBuilder
(
CommonServices
.
EnumPubSubServiceMode
serviceMode
)
{
public
PubSubServicePulsarBuilder
(
CommonServices
.
EnumPubSubServiceMode
serviceMode
)
{
this
.
serviceMode
=
serviceMode
;
this
.
serviceMode
=
serviceMode
;
...
@@ -345,6 +347,16 @@ public class ServiceBuilderFactory {
...
@@ -345,6 +347,16 @@ public class ServiceBuilderFactory {
return
this
;
return
this
;
}
}
public
PubSubServicePulsarBuilder
setAdminUrl
(
String
adminUrl
)
{
this
.
adminUrl
=
adminUrl
;
return
this
;
}
public
PubSubServicePulsarBuilder
setClusters
(
String
clusters
)
{
this
.
clusters
=
clusters
;
return
this
;
}
private
boolean
validateParams
()
{
private
boolean
validateParams
()
{
/**
/**
* defaulting to number of processors
* defaulting to number of processors
...
@@ -352,7 +364,7 @@ public class ServiceBuilderFactory {
...
@@ -352,7 +364,7 @@ public class ServiceBuilderFactory {
if
(
consumerPoolSize
==
0
){
if
(
consumerPoolSize
==
0
){
consumerPoolSize
=
Runtime
.
getRuntime
().
availableProcessors
();
consumerPoolSize
=
Runtime
.
getRuntime
().
availableProcessors
();
}
}
if
(
serviceUrl
==
null
)
if
(
serviceUrl
==
null
&&
adminUrl
!=
null
)
return
false
;
return
false
;
return
true
;
return
true
;
...
@@ -362,7 +374,7 @@ public class ServiceBuilderFactory {
...
@@ -362,7 +374,7 @@ public class ServiceBuilderFactory {
public
CommonServices
.
IService
build
()
{
public
CommonServices
.
IService
build
()
{
if
(
validateParams
())
{
if
(
validateParams
())
{
try
{
try
{
pubSubServicePulsar
=
new
IPubSubServicePulsarImpl
(
serviceUrl
,
consumerPoolSize
);
pubSubServicePulsar
=
new
IPubSubServicePulsarImpl
(
serviceUrl
,
adminUrl
,
clusters
,
consumerPoolSize
);
pubSubServicePulsar
.
setServiceMode
(
serviceMode
);
pubSubServicePulsar
.
setServiceMode
(
serviceMode
);
}
catch
(
Exception
exp
){
}
catch
(
Exception
exp
){
...
...
src/test/java/microservice/TestMicroserviceApp.java
View file @
fffb5b8a
...
@@ -103,6 +103,8 @@ public class TestMicroserviceApp {
...
@@ -103,6 +103,8 @@ public class TestMicroserviceApp {
.
build
();
.
build
();
CommonServices
.
IService
pulsarPubSub
=
ServiceBuilderFactory
.
createPubSubServicePulsarBuilder
(
CommonServices
.
EnumPubSubServiceMode
.
E_BOTH
)
CommonServices
.
IService
pulsarPubSub
=
ServiceBuilderFactory
.
createPubSubServicePulsarBuilder
(
CommonServices
.
EnumPubSubServiceMode
.
E_BOTH
)
.
setServiceUrl
(
"localhost:6650"
)
.
setServiceUrl
(
"localhost:6650"
)
.
setAdminUrl
(
"localhost:8080"
)
.
setClusters
(
"standalone"
)
.
build
();
.
build
();
microservice
.
MicroserviceApp
msApp
=
new
microservice
.
MicroserviceApp
(
appName
);
microservice
.
MicroserviceApp
msApp
=
new
microservice
.
MicroserviceApp
(
appName
);
...
@@ -169,12 +171,28 @@ public class TestMicroserviceApp {
...
@@ -169,12 +171,28 @@ public class TestMicroserviceApp {
(
msgCtx
,
orgService
)
->
{
(
msgCtx
,
orgService
)
->
{
testZmqMsgQueue
((
CommonServices
.
IMsgQService
.
MsgQContext
)
msgCtx
);
testZmqMsgQueue
((
CommonServices
.
IMsgQService
.
MsgQContext
)
msgCtx
);
}));
}));
methodParamsList
.
add
(
new
CommonServices
.
MethodParams
(
Enums
.
EnumServiceType
.
E_PUBSUB
,
CommonServices
.
EnumPubSubCommands
.
E_NOTIFY
,
"/activity"
,
(
msgCtx
,
orgService
)
->
{
testPubSubMsg
((
CommonServices
.
IPubSubService
.
PubSubMsgContext
)
msgCtx
);
}));
}
}
long
startTime
=
0
;
long
startTime
=
0
;
ObjectMapper
objectMapper
=
new
ObjectMapper
();
ObjectMapper
objectMapper
=
new
ObjectMapper
();
private
void
testPubSubMsg
(
CommonServices
.
IPubSubService
.
PubSubMsgContext
msgCtx
)
{
// System.out.println("Recieved mcid: " + msgCtx.getMcid());
try
{
JsonNode
jsonNode
=
objectMapper
.
readValue
(
msgCtx
.
getMsg
(),
JsonNode
.
class
);
handleJsonTestStartStop
(
jsonNode
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
/**
/**
* around 300k/s
* around 300k/s
* @param msgCtx
* @param msgCtx
...
@@ -182,6 +200,13 @@ public class TestMicroserviceApp {
...
@@ -182,6 +200,13 @@ public class TestMicroserviceApp {
private
void
testZmqMsgQueue
(
CommonServices
.
IMsgQService
.
MsgQContext
msgCtx
)
{
private
void
testZmqMsgQueue
(
CommonServices
.
IMsgQService
.
MsgQContext
msgCtx
)
{
try
{
try
{
JsonNode
jsonNode
=
objectMapper
.
readValue
(
msgCtx
.
msg
,
JsonNode
.
class
);
JsonNode
jsonNode
=
objectMapper
.
readValue
(
msgCtx
.
msg
,
JsonNode
.
class
);
handleJsonTestStartStop
(
jsonNode
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
private
void
handleJsonTestStartStop
(
JsonNode
jsonNode
)
{
String
state
=
jsonNode
.
path
(
"state"
).
asText
();
String
state
=
jsonNode
.
path
(
"state"
).
asText
();
switch
(
state
){
switch
(
state
){
case
"start"
:
case
"start"
:
...
@@ -193,9 +218,6 @@ public class TestMicroserviceApp {
...
@@ -193,9 +218,6 @@ public class TestMicroserviceApp {
case
"msg"
:
case
"msg"
:
break
;
break
;
}
}
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
}
private
void
testZmqRead
(
RestContext
msgCtx
)
{
private
void
testZmqRead
(
RestContext
msgCtx
)
{
...
...
src/test/java/microservice/TestPulsar.java
View file @
fffb5b8a
package
microservice
;
package
microservice
;
import
common.microservice.context.PubSubMsg
;
import
common.microservice.utils.IDGenerator
;
import
common.microservice.utils.IDGenerator
;
import
org.apache.pulsar.client.admin.PulsarAdmin
;
import
org.apache.pulsar.client.admin.PulsarAdmin
;
import
org.apache.pulsar.client.admin.PulsarAdminBuilder
;
import
org.apache.pulsar.client.admin.PulsarAdminBuilder
;
...
@@ -9,6 +10,7 @@ import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl;
...
@@ -9,6 +10,7 @@ import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl;
import
org.apache.pulsar.client.api.*
;
import
org.apache.pulsar.client.api.*
;
import
org.apache.pulsar.client.api.url.URL
;
import
org.apache.pulsar.client.api.url.URL
;
import
org.apache.pulsar.client.impl.conf.ClientConfigurationData
;
import
org.apache.pulsar.client.impl.conf.ClientConfigurationData
;
import
org.apache.pulsar.client.impl.schema.JSONSchema
;
import
org.apache.pulsar.common.policies.data.Policies
;
import
org.apache.pulsar.common.policies.data.Policies
;
import
org.apache.pulsar.common.policies.data.TenantInfo
;
import
org.apache.pulsar.common.policies.data.TenantInfo
;
import
org.junit.Test
;
import
org.junit.Test
;
...
@@ -28,18 +30,20 @@ public class TestPulsar {
...
@@ -28,18 +30,20 @@ public class TestPulsar {
public
void
testTopicProducer
()
throws
PulsarClientException
{
public
void
testTopicProducer
()
throws
PulsarClientException
{
String
localClusterUrl
=
"pulsar://localhost:6650"
;
String
localClusterUrl
=
"pulsar://localhost:6650"
;
String
namespace
=
"mcx/testApp"
;
// This namespace is created automatically
String
namespace
=
"mcx/testApp"
;
// This namespace is created automatically
String
topic
=
String
.
format
(
"persistent://%s/
my-topic1
"
,
namespace
);
String
topic
=
String
.
format
(
"persistent://%s/
activity
"
,
namespace
);
PulsarClient
client
=
PulsarClient
.
builder
()
PulsarClient
client
=
PulsarClient
.
builder
()
.
serviceUrl
(
localClusterUrl
)
.
serviceUrl
(
localClusterUrl
)
.
build
();
.
build
();
Producer
producer
=
client
.
newProducer
().
topic
(
topic
).
create
();
Producer
<
PubSubMsg
>
producer
=
client
.
newProducer
(
JSONSchema
.
of
(
PubSubMsg
.
class
))
.
topic
(
topic
)
.
create
();
// Publish 10 messages to the topic
// Publish 10 messages to the topic
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
for
(
int
i
=
0
;
i
<
100
;
i
++)
{
producer
.
send
(
String
.
format
(
"Message number %d"
,
i
).
getBytes
());
PubSubMsg
pubSubMsg
=
new
PubSubMsg
(
"hello - "
+
String
.
valueOf
(
System
.
currentTimeMillis
()),
String
.
valueOf
(
i
));
producer
.
send
(
pubSubMsg
);
System
.
out
.
println
(
"Sending message"
);
System
.
out
.
println
(
"Sending message"
);
// final Message message = MessageBuilder.create()
// final Message message = MessageBuilder.create()
// .setContent(String.format("Message number %d", i).getBytes())
// .setContent(String.format("Message number %d", i).getBytes())
...
@@ -65,17 +69,24 @@ public class TestPulsar {
...
@@ -65,17 +69,24 @@ public class TestPulsar {
//final Consumer consumer = client.newConsumer().subscriptionName("my-sub-1").topic(topic).subscribe();
//final Consumer consumer = client.newConsumer().subscriptionName("my-sub-1").topic(topic).subscribe();
Pattern
allTopicsInNamespace
=
Pattern
.
compile
(
"persistent://mcx/testApp/.*"
);
Pattern
allTopicsInNamespace
=
Pattern
.
compile
(
"persistent://mcx/testApp/.*"
);
final
Consumer
consumer
=
client
.
newConsumer
(
)
final
Consumer
<
PubSubMsg
>
consumer
=
client
.
newConsumer
(
JSONSchema
.
of
(
PubSubMsg
.
class
)
)
.
subscriptionName
(
IDGenerator
.
createUUID
())
.
subscriptionName
(
IDGenerator
.
createUUID
())
//
.subscriptionType(SubscriptionType.Shared) // enable for multi-instance
.
subscriptionType
(
SubscriptionType
.
Shared
)
// enable for multi-instance
.
topicsPattern
(
allTopicsInNamespace
)
.
topicsPattern
(
allTopicsInNamespace
)
.
subscribe
();
.
subscribe
();
// final Consumer consumer = client.newConsumer()
// .subscriptionName(IDGenerator.createUUID())
// .subscriptionType(SubscriptionType.Shared) // enable for multi-instance
// .topicsPattern(allTopicsInNamespace)
// .subscribe();
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
// Wait for a message
// Wait for a message
Message
msg
=
consumer
.
receive
();
Message
<
PubSubMsg
>
msg
=
consumer
.
receive
();
System
.
out
.
printf
(
"Message received: %s\n"
,
new
String
(
msg
.
getData
()
));
System
.
out
.
printf
(
"Message received: %s\n"
,
msg
.
getValue
().
toString
(
));
// Acknowledge the message so that it can be deleted by the message broker
// Acknowledge the message so that it can be deleted by the message broker
consumer
.
acknowledge
(
msg
);
consumer
.
acknowledge
(
msg
);
...
...
src/test/java/microservice/TestServicesAndMethods.java
View file @
fffb5b8a
...
@@ -36,7 +36,7 @@ import java.util.stream.Collectors;
...
@@ -36,7 +36,7 @@ import java.util.stream.Collectors;
*/
*/
public
class
TestServicesAndMethods
{
public
class
TestServicesAndMethods
{
public
static
final
int
ITERATIONS
=
10000
000
;
public
static
final
int
ITERATIONS
=
10000
;
private
static
class
RoutingMatch
{
private
static
class
RoutingMatch
{
...
@@ -278,12 +278,25 @@ public class TestServicesAndMethods {
...
@@ -278,12 +278,25 @@ public class TestServicesAndMethods {
@Test
@Test
public
void
testPubSubPulsar
()
throws
InterruptedException
{
public
void
testPubSubPulsar
()
throws
InterruptedException
{
CommonServices
.
IService
iService
=
ServiceBuilderFactory
.
createPubSubServicePulsarBuilder
(
CommonServices
.
EnumPubSubServiceMode
.
E_
BOTH
)
CommonServices
.
IService
iService
=
ServiceBuilderFactory
.
createPubSubServicePulsarBuilder
(
CommonServices
.
EnumPubSubServiceMode
.
E_
PUBLISHER
)
.
setServiceUrl
(
"localhost:6650"
)
.
setServiceUrl
(
"localhost:6650"
)
.
setAdminUrl
(
"localhost:8080"
)
.
build
();
.
build
();
CommonServices
.
IPubSubService
pubSubService
=
(
CommonServices
.
IPubSubService
)
iService
;
CommonServices
.
IPubSubService
pubSubService
=
(
CommonServices
.
IPubSubService
)
iService
;
// pubSubService.init();
pubSubService
.
init
();
// pubSubService.run();
pubSubService
.
run
();
ObjectNode
objectNode
=
JsonNodeFactory
.
instance
.
objectNode
().
put
(
"state"
,
"start"
).
put
(
"iterations"
,
ITERATIONS
);
System
.
out
.
println
(
"Testing "
+
String
.
valueOf
(
ITERATIONS
)
+
" iterations"
);
long
start
=
System
.
currentTimeMillis
();
pubSubService
.
publish
(
new
CommonServices
.
IPubSubService
.
PubSubMsgContext
(
"/activity"
,
objectNode
.
toString
()));
objectNode
.
put
(
"state"
,
"msg"
);
for
(
int
i
=
0
;
i
<
TestServicesAndMethods
.
ITERATIONS
;
i
++)
{
objectNode
.
put
(
"msg"
,
"hello"
+
String
.
valueOf
(
i
));
pubSubService
.
publish
(
new
CommonServices
.
IPubSubService
.
PubSubMsgContext
(
"/activity"
,
objectNode
.
toString
()));
}
objectNode
.
put
(
"state"
,
"end"
);
pubSubService
.
publish
(
new
CommonServices
.
IPubSubService
.
PubSubMsgContext
(
"/activity"
,
objectNode
.
toString
()));
System
.
out
.
println
(
"Async publish Test of: "
+
String
.
valueOf
(
ITERATIONS
)
+
" took (msec): "
+
String
.
valueOf
(
System
.
currentTimeMillis
()
-
start
));
Thread
.
sleep
(
1000
);
Thread
.
sleep
(
1000
);
pubSubService
.
shutdown
();
pubSubService
.
shutdown
();
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment