houshanren/SkyEye

sudo bash build.sh 1.3.0 master

git clone https://github.com/JThink/dubbox.git
cd dubbox
git checkout skyeye-trace-1.3.0
修改相关pom中的私服地址
mvn clean install deploy -Dmaven.test.skip=true

mysql -uroot -p
source skyeye-data/skyeye-data-jpa/src/main/resources/sql/init.sql

hbase shell
执行skyeye-collector/skyeye-collector-trace/src/main/resources/shell/hbase这个文件里面的内容

cd skyeye-collector/skyeye-collector-indexer/src/main/resources/shell
./install.sh
bash start.sh app-log http://192.168.xx.xx:9200,http://192.168.xx.xx:9200,......
cd skyeye-collector/skyeye-collector-metrics/src/main/resources/shell
bash start.sh event-log http://192.168.xx.xx:9200,http://192.168.xx.xx:9200,......

注意点：如果es版本为5.x，那么需要修改skyeye-collector/src/main/resources/shell/es/app-log/create-index.py的49和50行为下面内容：
'messageSmart': { 'type': 'text', 'analyzer': 'ik_smart', 'search_analyzer': 'ik_smart', 'include_in_all': 'true', 'boost': 8},
'messageMax': { 'type': 'text', 'analyzer': 'ik_max_word', 'search_analyzer': 'ik_max_word', 'include_in_all': 'true', 'boost': 8}

kafka-topics.sh --create --zookeeper 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181/kafka/0.10.0.1 --replication-factor 3 --partitions 9 --topic app-log

./zkCli.sh
执行skyeye-monitor/src/main/resources/shell/zk这个文件里面的内容

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/alarm
vim alarm.properties

# log_mailer request queue
rabbit.request.addresses=localhost:5672
rabbit.request.username=jthink
rabbit.request.password=jthink
rabbit.request.vhost=/dev
rabbit.request.channelCacheSize=50
rabbit.request.queue=log_mailer
rabbit.request.exchange=direct.log
rabbit.request.routingKey=log.key

# mail
mail.jthink.smtphost=smtp.xxx.com
mail.jthink.port=25
[email protected]
[email protected]
mail.jthink.password=jthink_0926

cd skyeye-alarm
gradle clean distZip -x test
cd target/distributions
unzip skyeye-alarm-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-alarm-x.x.x
nohup bin/skyeye-alarm &

cd skyeye-collector
gradle clean build -x test

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/collector
vim collector-backup.properties

# kafka config
kafka.brokers=riot01:9092,riot02:9092,riot03:9092
kafka.topic=app-log
kafka.consume.group=log-backup-consume-group
kafka.poll.timeout=100

# hdfs
hadoop.hdfs.namenode.port=8020
hadoop.hdfs.namenode.host=192.168.88.131
hadoop.hdfs.user=xxx
hadoop.hdfs.baseDir=/user/xxx/JThink/
hadoop.hdfs.fileRoot=/tmp/monitor-center/
upload.log.cron=0 30 0 * * ?

cd skyeye-collector-backup/target/distributions
unzip skyeye-collector-backup-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-collector-backup-x.x.x
nohup bin/skyeye-collector-backup &

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/collector
vim collector-indexer.properties

# kafka config
kafka.brokers=riot01:9092,riot02:9092,riot03:9092
kafka.topic=app-log
kafka.consume.group=es-indexer-consume-group
kafka.poll.timeout=100

# es config
es.ips=riot01,riot02,riot03
es.cluster=mondeo
es.port=9300
es.sniff=true
es.index=app-log
es.doc=log

cd skyeye-collector-indexer/target/distributions
unzip skyeye-collector-indexer-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-collector-indexer-x.x.x
nohup bin/skyeye-collector-indexer &

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/collector
vim collector-metrics.properties

# kafka config
kafka.brokers=riot01:9092,riot02:9092,riot03:9092
kafka.topic=app-log
kafka.consume.group=info-collect-consume-group
kafka.poll.timeout=100

# es config
es.ips=riot01,riot02,riot03
es.cluster=mondeo
es.port=9300
es.sniff=true
es.index=event-log
es.doc=log

# redis config
redis.host=localhost
redis.port=6379
redis.password=

# mysql config
database.address=localhost:3306
database.name=monitor-center
database.username=root
database.password=root

# log_mailer request queue
rabbit.request.addresses=localhost:5672
rabbit.request.username=jthink
rabbit.request.password=jthink
rabbit.request.vhost=/dev
rabbit.request.channelCacheSize=50
rabbit.request.queue=log_mailer
rabbit.request.exchange=direct.log
rabbit.request.routingKey=log.key

# zk
zookeeper.zkServers=riot01:2181,riot02:2181,riot03:2181
zookeeper.sessionTimeout=60000
zookeeper.connectionTimeout=5000

cd skyeye-collector-metrics/target/distributions
unzip skyeye-collector-metrics-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-collector-metrics-x.x.x
nohup bin/skyeye-collector-metrics &

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/collector
vim collector-trace.properties

# kafka config
kafka.brokers=riot01:9092,riot02:9092,riot03:9092
kafka.topic=app-log
kafka.consume.group=rpc-trace-consume-group
kafka.poll.timeout=100

# redis config
redis.host=localhost
redis.port=6379
redis.password=

# mysql config
database.address=localhost:3306
database.name=monitor-center
database.username=root
database.password=root

# hbase config
hbase.quorum=panda-01,panda-01,panda-03
hbase.rootDir=hdfs://panda-01:8020/hbase
hbase.zookeeper.znode.parent=/hbase

cd skyeye-collector-trace/target/distributions
unzip skyeye-collectortracemetrics-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-collector-trace-x.x.x
nohup bin/skyeye-collector-trace &

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/monitor
vim monitor.properties

# zk
zookeeper.zkServers=riot01:2181,riot02:2181,riot03:2181
zookeeper.sessionTimeout=60000
zookeeper.connectionTimeout=5000
zookeeper.baseSleepTimeMs=1000
zookeeper.maxRetries=3

# log_mailer request queue
rabbit.request.addresses=localhost:5672
rabbit.request.username=jthink
rabbit.request.password=jthink
rabbit.request.vhost=/dev
rabbit.request.channelCacheSize=50
rabbit.request.queue=log_mailer
rabbit.request.exchange=direct.log
rabbit.request.routingKey=log.key

# mysql config
database.address=localhost:3306
database.name=monitor-center
database.username=root
database.password=root

cd skyeye-monitor
gradle clean distZip -x test
cd target/distributions
unzip skyeye-monitor-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-monitor-x.x.x
nohup bin/skyeye-monitor &

ssh 到部署节点
mkdir -p /opt/jthink/jthink-config/skyeye/web
vim web.properties

# server
serverAddress=0.0.0.0
serverPort=8090

# mysql config
database.address=localhost:3306
database.name=monitor-center
database.username=root
database.password=root

# es sql url
es.sql.url=http://riot01:9200/_sql?sql=
es.sql.sql=select * from app-log/log
es.query.delay=10
es.sql.index.event=event-log/log

# log_mailer request queue
rabbit.request.addresses=localhost:5672
rabbit.request.username=jthink
rabbit.request.password=jthink
rabbit.request.vhost=/dev
rabbit.request.channelCacheSize=50
rabbit.request.queue=log_mailer
rabbit.request.exchange=direct.log
rabbit.request.routingKey=log.key

# monitor
monitor.es.interval=0 */1 * * * ?					# 监控代码执行的周期，建议不修改
[email protected]

# hbase config
hbase.quorum=panda-01,panda-01,panda-03
hbase.rootDir=hdfs://panda-01:8020/hbase
hbase.zookeeper.znode.parent=/hbase

cd skyeye-web
gradle clean distZip -x test
cd target/distributions
unzip skyeye-web-x.x.x.zip(替换相应的x为自己的版本)

cd skyeye-web-x.x.x
nohup bin/skyeye-web &

compile "skyeye:skyeye-client-logback:1.3.0"

<property name="APP_NAME" value="your-app-name" />
<!-- kafka appender -->
<appender name="kafkaAppender" class="com.jthink.skyeye.client.logback.appender.KafkaAppender">
    <encoder class="com.jthink.skyeye.client.logback.encoder.KafkaLayoutEncoder">
      <layout class="ch.qos.logback.classic.PatternLayout">
        <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS};${CONTEXT_NAME};HOSTNAME;%thread;%-5level;%logger{96};%line;%msg%n</pattern>
      </layout>
    </encoder>
    <topic>app-log</topic>
    <rpc>none</rpc>
    <zkServers>riot01.jthink.com:2181,riot02.jthink.com:2181,riot03.jthink.com:2181</zkServers>
    <mail>[email protected]</mail>
    <keyBuilder class="com.jthink.skyeye.client.logback.builder.AppHostKeyBuilder" />

    <config>bootstrap.servers=riot01.jthink.com:9092,riot02.jthink.com:9092,riot03.jthink.com:9092</config>
    <config>acks=0</config>
    <config>linger.ms=100</config>
    <config>max.block.ms=5000</config>
  </appender>

compile "skyeye:skyeye-client-log4j:1.3.0"

<appender name="kafkaAppender" class="com.jthink.skyeye.client.log4j.appender.KafkaAppender">
        <param name="topic" value="app-log"/>
        <param name="zkServers" value="riot01.jthink.com:2181,riot02.jthink.com:2181,riot03.jthink.com:2181"/>
        <param name="app" value="xxx"/>
        <param name="rpc" value="dubbo"/>
        <param name="mail" value="[email protected]"/>
        <param name="bootstrapServers" value="riot01.jthink.com:9092,riot02.jthink.com:9092,riot03.jthink.com:9092"/>
        <param name="acks" value="0"/>
        <param name="maxBlockMs" value="2000"/>
        <param name="lingerMs" value="100"/>

        <layout class="org.apache.log4j.PatternLayout">
            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss.SSS};APP_NAME;HOSTNAME;%t;%p;%c;%L;%m%n"/>
        </layout>
    </appender>

compile "skyeye:skyeye-client-log4j2:1.3.0"

<KafkaCustomize name="KafkaCustomize" topic="app-log" zkServers="riot01.jthink.com:2181,riot02.jthink.com:2181,riot03.jthink.com:2181"
                mail="[email protected]" rpc="none" app="${APP_NAME}">
  <ThresholdFilter level="info" onMatch="ACCEPT" onMismatch="DENY"/>
  <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS};${APP_NAME};HOSTNAME;%t;%-5level;%logger{96};%line;%msg%n"/>
  <Property name="bootstrap.servers">riot01.jthink.com:9092,riot02.jthink.com:9092,riot03.jthink.com:9092</Property>
  <Property name="acks">0</Property>
  <Property name="linger.ms">100</Property>
</KafkaCustomize>

compile "com.101tec:zkclient:0.10"
compile ("com.alibaba:dubbo:2.8.4-skyeye-trace-1.3.0") {
  exclude group: 'org.springframework', module: 'spring'
}

LOGGER.info("我是测试日志打印")

// 参数依次为EventType(事件类型)、api、账号、请求耗时、成功还是失败、具体自定义的日志内容
LOGGER.info(ApiLog.buildApiLog(EventType.invoke_interface, "/app/status", "800001", 100, EventLog.MONITOR_STATUS_SUCCESS, "我是mock api成功日志").toString());
LOGGER.info(ApiLog.buildApiLog(EventType.invoke_interface, "/app/status", "800001", 10, EventLog.MONITOR_STATUS_FAILED, "我是mock api失败日志").toString());

// 参数依次为EventType(事件类型)、MiddleWare(中间件名称)、操作耗时、成功还是失败、具体自定义的日志内容
LOGGER.info(EventLog.buildEventLog(EventType.middleware_opt, MiddleWare.HBASE.symbol(), 100, EventLog.MONITOR_STATUS_SUCCESS, "我是mock middle ware成功日志").toString());
LOGGER.info(EventLog.buildEventLog(EventType.middleware_opt, MiddleWare.MONGO.symbol(), 10, EventLog.MONITOR_STATUS_FAILED, "我是mock middle ware失败日志").toString());

// job执行仅仅处理失败的日志（成功的不做处理，所以只需要构造失败的日志）, 参数依次为EventType(事件类型)、job 的id号、操作耗时、失败、具体自定义的日志内容
LOGGER.info(EventLog.buildEventLog(EventType.job_execute, "application_1477705439920_0544", 10, EventLog.MONITOR_STATUS_FAILED, "我是mock job exec失败日志").toString());

// 参数依次为EventType(事件类型)、第三方名称、操作耗时、成功还是失败、具体自定义的日志内容
LOGGER.info(EventLog.buildEventLog(EventType.thirdparty_call, "xx1", 100, EventLog.MONITOR_STATUS_FAILED, "我是mock third 失败日志").toString());
LOGGER.info(EventLog.buildEventLog(EventType.thirdparty_call, "xx1", 100, EventLog.MONITOR_STATUS_SUCCESS, "我是mock third 成功日志").toString());
LOGGER.info(EventLog.buildEventLog(EventType.thirdparty_call, "xx2", 100, EventLog.MONITOR_STATUS_SUCCESS, "我是mock third 成功日志").toString());
LOGGER.info(EventLog.buildEventLog(EventType.thirdparty_call, "xx2", 100, EventLog.MONITOR_STATUS_FAILED, "我是mock third 失败日志").toString());