第 1 层  热记忆（每轮注入，固定前缀，命中 KV 缓存）
         ~/.hermes/memories/MEMORY.md   ≤2200 字符
         ~/.hermes/memories/USER.md     ≤1375 字符
         ▲ 模型调用 memory 工具维护；写满即报错要求合并

第 2 层  冷记忆（本地事实库，按需检索，不占提示词固定开销）
         ~/.hermes/memory_store.db  (SQLite: facts 表)
         ▲ 模型 memory add 时自动镜像落库（0 额外 token）
         ▲ 定时抽取脚本补录"该记但没记"的事实

第 3 层  会话存档（全文，可检索）
         ~/.hermes/state.db  (messages 表)
         ▲ FTS5 全文检索旧对话，不花模型费用

定时器 A  每周日 23:00   holo_extract.py           非思考模型   7 天窗口   快速抄录
定时器 B  每月 1 号 23:00 holo_monthly_sweep.sh    思考模型+--prefs  30 天窗口 补漏 + 体检

① 盘点真实技能名 → 走文件系统（~/.hermes/skills 递归、排除 .archive）
                 ⚠️ 绝对不要用 `hermes skills list` 表格取名字（会被截断成 audiocraft-audi…）
② 先验证恢复通道 → 挑一个目标：uninstall → repair-official --restore 走通一遍
③ 分类          → ① 用户直接相关 ② 明确无关 ③ 边界类（先问用户）
④ 批量执行      → 逐条记录结果（成功/失败/原因）
⑤ 删完复测      → prompt-size 用数字证明索引确实变小

ssh 192.168.2.180 -l mi

sudo apt install tightvncserver

vncpasswd

Would you like to enter a view-only password (y/n)? n

tightvncserver

tightvncserver -kill :1

vim ~/.vnc/xstartup

#!/bin/sh
xsetroot -solid grey
export XKL_XMODMAP_DISABLE=1
#autocutsel -fork
openbox &
/usr/bin/lxsession -s Lubuntu &

tightvncserver :1

cat ~/.vnc/lubuntu\:1.log

sudo vim /etc/xdg/lxsession/Lubuntu/autostart

@/usr/bin/tightvncserver :1

/usr/bin/tightvncserver :1

sudo vi /usr/bin/vncserver

#关闭
tightvncserver -kill :1
#启动
tightvncserver :1

from superset.custom_security import CustomSecurityManager
CUSTOM_SECURITY_MANAGER = CustomSecurityManager

109             <li><a href="{{appbuilder.get_url_for_userinfo}}"><span class="fa fa-fw fa-user"></span>{{_("Profile")}}</a></li>
110             <li><a href="/sso/logout"><span class="fa fa-fw fa-sign-out"></span>{{_("Logout")}}</a></li>
111             <!--<li><a href="{{appbuilder.get_url_for_logout}}"><span class="fa fa-fw fa-sign-out"></span>{{_("Logout")}}</a></li>-->
123 {% else %}
124     <li><a href="/sso/login">
125     <!--<li><a href="{{appbuilder.get_url_for_login}}">-->
126     <i class="fa fa-fw fa-sign-in"></i>SSO_{{_("Login")}}</a></li>
127 {% endif %}

from flask import redirect, g, flash, request
from flask_appbuilder.security.views import UserDBModelView,AuthDBView
from superset.security import SupersetSecurityManager
from flask_appbuilder.security.views import expose
from flask_appbuilder.security.manager import BaseSecurityManager
from flask_login import login_user, logout_user
import requests
import superset_config
class CustomAuthDBView(AuthDBView):
    @expose('/sso/login/', methods=['GET'])
    def sso_login(self):
        print("------------ login", request.args)
        if request.args.get('ticket') is not None:
            r = requests.get(**login_url**, verify=False)
            print("------------ validate", r.json()['username'])
            username = r.json()['username']
            user = self.appbuilder.sm.find_user(username=username)
            if user is not None:
                print("------------ find user", user)
                login_user(user, remember=False)
                return redirect(self.appbuilder.get_url_for_index)
            else:
                user = self.appbuilder.sm.add_user(
                    username=username,
                    first_name=username.split('.')[0],
                    last_name=username.split('.')[1],
                    password = username,
                    email=username,
                    role=self.appbuilder.sm.find_role(self.appbuilder.sm.auth_user_registration_role))
                print("------------ new user", user)
                login_user(user, remember=False)
                return redirect(self.appbuilder.get_url_for_index)
        else:
            return redirect(***login_url***)
    @expose('/sso/logout/', methods=['GET'])
    def sso_logout(self):
        print("------------ logout", logout_url)
        return redirect(**logout_url**)
class CustomSecurityManager(SupersetSecurityManager):
    authdbview = CustomAuthDBView
    def __init__(self, appbuilder):
        super(CustomSecurityManager, self).__init__(appbuilder)

CACHE_CONFIG = {
    'CACHE_TYPE': 'redis',
    'CACHE_DEFAULT_TIMEOUT': 60 * 60 * 24, # 1 day default (in secs)
    'CACHE_KEY_PREFIX': 'superset_results',
    'CACHE_REDIS_URL': 'redis://localhost:6379/0',
}

BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
CELERY_ACKS_LATE = True
#关闭utc时区，添加上海时区
CELERY_ENABLE_UTC = False
CELERY_TIMEZONE = 'Asia/Shanghai'

class CeleryConfig(object):
    BROKER_URL = 'redis://localhost:6379/0'
    CELERY_IMPORTS = (
        'superset.sql_lab',
        'superset.tasks',
    )
    CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
    CELERYD_LOG_LEVEL = 'DEBUG'
    CELERYD_PREFETCH_MULTIPLIER = 10
    CELERY_ACKS_LATE = True
    CELERY_ENABLE_UTC = False
    CELERY_TIMEZONE = 'Asia/Shanghai'
    CELERY_ANNOTATIONS = {
        'sql_lab.get_sql_results': {
            'rate_limit': '100/s',
        },
        'email_reports.send': {
            'rate_limit': '1/s',
            'time_limit': 120,
            'soft_time_limit': 150,
            'ignore_result': True,
        },
    }
    CELERYBEAT_SCHEDULE = {
        'cache-warmup-hourly': {
            'task': 'cache-warmup',
            'schedule': crontab(minute=0,hour='*/2'),  # hourly
            'kwargs': {
                'strategy_name': 'top_n_dashboards',
                'top_n': 10,
                'since': '7 days ago',
            },
        },
    }

CELERY_CONFIG = CeleryConfig

CELERYBEAT_SCHEDULE = {
    'cache-warmup-hourly': {
        'task': 'cache-warmup',
        'schedule': crontab(minute=0, hour='*'),  # hourly
        'kwargs': {
            'strategy_name': 'top_n_dashboards',
            'top_n': 5,
            'since': '7 days ago',
        },
    },
}

SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
SUPERSET_WEBSERVER_PORT = 7668

[Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/gunicorn/util.py", line 135, in load_class
    mod = import_module('.'.join(components))
  File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
   File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 724, in exec_module
  File "<frozen importlib._bootstrap_external>", line 860, in get_code
  File "<frozen importlib._bootstrap_external>", line 791, in source_to_code
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/geventlet.py", line 27
    from gunicorn.workers.async import AsyncWorker
                              ^
SyntaxError: invalid syntax
]

#-D 后台执行
#-w 启动进程数
#--log-level debug 日志级别
#--access-logfile /tmp/superset_access.log 指定access请求日志路径
#--error-logfile /tmp/superset_error.log 指定请求错误日志路径

前台启动：
gunicorn  -w 10 -k gevent --timeout 120 -b 0.0.0.0:7668 --limit-request-line 0 --limit-request-field_size 0 "superset.app:create_app()"

后台执行：
superset 0.35.0
gunicorn \
    -w 10 \
    --timeout 120 \
    -b 0.0.0.0:7668 \
    --limit-request-line 0 \
    --limit-request-field_size 0 \
    --forwarded-allow-ips=*
    --log-level debug
    --access-logfile /tmp/superset_access.log
    --error-logfile /tmp/superset_error.log
    superset:app -D

superset 0.36.0
gunicorn \
    -w 10 \
    --timeout 120 \
    -b 0.0.0.0:7668 \
    --limit-request-line 0 \
    --limit-request-field_size 0 \
    --forwarded-allow-ips=*
    --log-level debug
    --access-logfile /tmp/superset_access.log
    --error-logfile /tmp/superset_error.log
    "superset.app:create_app()" -D

[default]
aws_access_key_id = your_access_key_id
aws_secret_access_key = your_secret_access_key

[default]
region = your_aws_region

 Superset 是一款由 Airbnb 开源的“现代化的企业级 BI（商业智能） Web 应用程序”，其通过创建和分享 dashboard，为数据分析提供了轻量级的数据查询和可视化方案。
 Superset 的前端主要用到了 React 和 NVD3/D3，而后端则基于 Python 的 Flask 框架和 Pandas、SQLAlchemy 等依赖库，主要提供了这几方面的功能：
集成数据查询功能，支持多种数据库，包括 MySQL、PostgresSQL、Oracle、SQL Server、SQLite、SparkSQL 等，并深度支持 Druid。
通过 NVD3/D3 预定义了多种可视化图表，满足大部分的数据展示功能。如果还有其他需求，也可以自开发更多的图表类型，或者嵌入其他的 JavaScript 图表库（如 HighCharts、ECharts）。
提供细粒度安全模型，可以在功能层面和数据层面进行访问控制。支持多种鉴权方式（如数据库、OpenID、LDAP、OAuth、REMOTE_USER 等）。

git clone https://github.com/apache/incubator-superset.git

git tag　列出所有版本号

#切换到自己想要的版本
git checkout　+某版本号

#进入到目录
cd incubator-superset
pip install -r requirements.txt -i https://pypi.douban.com/simple/

安装开发依赖
pip install -r requirements-dev.txt -i https://pypi.douban.com/simple/

#incubator-superset 目录执行
pip install -e .

创建管理员账户和密码
这个命令已经不推荐使用了。
fabmanager create-admin --app superset
推荐使用
$export FLASK_APP=superset
flask fab create-admin

#初始化数据库
python superset db upgrade

#初始化角色和权限
python superset init

#加载示例数据
python superset load_examples

ln -s /xxxx/nodejs/bin/node /usr/local/bin/
ln -s /xxxxx/nodejs/bin/npm /usr/local/bin/ 

验证 `npm config get registry`

npm run build
npm run dev

from superset import app
app.run(debug=True, host='0.0.0.0', port=8088)

#
# This file is autogenerated by pip-compile
# To update, run:
#
#    pip-compile --output-file=requirements.txt setup.py
#
alembic==1.0.11           # via flask-migrate
amqp==2.5.0               # via kombu
apispec[yaml]==1.3.3      # via flask-appbuilder
asn1crypto==0.24.0        # via cryptography
attrs==19.1.0             # via jsonschema
babel==2.7.0              # via flask-babel
billiard==3.6.0.0         # via celery
bleach==3.1.0
celery==4.3.0
cffi==1.12.3              # via cryptography
click==6.7
colorama==0.4.1
contextlib2==0.5.5
croniter==0.3.30
cryptography==2.7
decorator==4.4.0          # via retry
defusedxml==0.6.0         # via python3-openid
flask-appbuilder==2.1.13
flask-babel==0.12.2       # via flask-appbuilder
flask-caching==1.7.2
flask-compress==1.4.0
flask-jwt-extended==3.20.0  # via flask-appbuilder
flask-login==0.4.1        # via flask-appbuilder
flask-migrate==2.5.2
flask-openid==1.2.5       # via flask-appbuilder
flask-sqlalchemy==2.4.0   # via flask-appbuilder, flask-migrate
flask-talisman==0.7.0
flask-wtf==0.14.2
flask==1.1.1
future==0.17.1            # via parsedatetime
geographiclib==1.49       # via geopy
geopy==1.20.0
gunicorn==19.8.1
humanize==0.5.1
idna==2.8
isodate==0.6.0
itsdangerous==1.1.0       # via flask
jinja2==2.10.1            # via flask, flask-babel
jsonschema==3.0.1         # via flask-appbuilder
kombu==4.6.3              # via celery
mako==1.0.14              # via alembic
markdown==3.1.1
markupsafe==1.1.1         # via jinja2, mako
marshmallow-enum==1.4.1   # via flask-appbuilder
marshmallow-sqlalchemy==0.17.0  # via flask-appbuilder
marshmallow==2.19.5       # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
numpy==1.17.0             # via pandas
pandas==0.24.2
parsedatetime==2.4
pathlib2==2.3.4
polyline==1.4.0
prison==0.1.2             # via flask-appbuilder
py==1.8.0                 # via retry
pycparser==2.19           # via cffi
pyjwt==1.7.1              # via flask-appbuilder, flask-jwt-extended
pyrsistent==0.15.4        # via jsonschema
python-dateutil==2.8.0
python-dotenv==0.10.3
python-editor==1.0.4      # via alembic
python-geohash==0.8.5
python3-openid==3.1.0     # via flask-openid
pytz==2019.2              # via babel, celery, pandas
pyyaml==5.1.2
retry==0.9.2
selenium==3.141.0
simplejson==3.16.0
six==1.12.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
sqlalchemy-utils==0.34.1
sqlalchemy==1.3.6
sqlparse==0.3.0
urllib3==1.25.3           # via selenium
vine==1.3.0               # via amqp, celery
webencodings==0.5.1       # via bleach
werkzeug==0.15.5          # via flask, flask-jwt-extended
wtforms-json==0.3.3
wtforms==2.2.1            # via flask-wtf, wtforms-json

#创建管理用户及密码
$ export FLASK_APP=superset
flask fab create-admin
或者
fabmanager create-admin --app superset

#初始化数据库
superset db upgrade

#初始化角色和权限
superset init

#加载示例数据
superset load_examples
superset load_examples

#debug模式启动，并指定8088端口
superset run -p 8088 --with-threads --reload --debugger

2020-02-21 18:16:07,701:INFO:root:Configured event logger of type <class 'superset.utils.log.DBEventLogger'>
Recognized Database Authentications.
2020-02-21 18:16:08,948:ERROR:flask_appbuilder.security.sqla.manager:Error adding new user to database. (sqlite3.IntegrityError) UNIQUE constraint failed: ab_user.email
[SQL: INSERT INTO ab_user (first_name, last_name, username, password, active, email, last_login, login_count, fail_login_count, created_on, changed_on, created_by_fk, changed_by_fk) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)]
[parameters: ('admin', 'user', 'admin', 'pbkdf2:sha256:150000$lXoRkj1U$96d4f6c545b94200fabf2e0cee44e66ba928982109d926c75633c10db0a78d7a', 1, 'admin@superset.com', None, None, None, '2020-02-21 18:16:08.946516', '2020-02-21 18:16:08.946527', None, None)]
(Background on this error at: http://sqlalche.me/e/gkpj)
No user created an error occured

The "loose" option must be the same for @babel/plugin-proposal-class-properties, @babel/plugin-proposal-private-methods and @babel/plugin-proposal-private-property-in-object (when they are enabled): you can silence this warning by explicitly adding

42   plugins: [
43     'lodash',
44     '@babel/plugin-syntax-dynamic-import',
45     //'@babel/plugin-proposal-class-properties',
46     '@babel/plugin-proposal-optional-chaining',
47     ['@babel/plugin-transform-runtime', { corejs: 3 }],
48     'react-hot-loader/babel',
49      ['@babel/plugin-proposal-private-methods', { 'loose': true }],
50      ['@babel/plugin-proposal-class-properties', { 'loose': true }]
51   ],

ERROR in /data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/src/components/ListView/TableCollection.tsx
ERROR in /data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/src/components/ListView/TableCollection.tsx(45,22):
TS2339: Property 'hidden' does not exist on type 'HeaderGroup<{}>'.

ERROR in /data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/src/components/ListView/TableCollection.tsx
ERROR in /data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/src/components/ListView/TableCollection.tsx(48,28):
TS2339: Property 'sortable' does not exist on type 'HeaderGroup<{}>'.

ERROR in /data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/src/components/ListView/TableCollection.tsx
ERROR in /data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/src/components/ListView/TableCollection.tsx(54,27):
TS2339: Property 'sortable' does not exist on type 'HeaderGroup<{}>'.

Error from chokidar (/data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/node_modules/fbjs/lib): Error: ENOSPC: System limit for number of file watchers reached, watch '/data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/node_modules/fbjs/lib/camelizeStyleName.js'
Error from chokidar (/data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/node_modules/fbjs/lib): Error: ENOSPC: System limit for number of file watchers reached, watch '/data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/node_modules/fbjs/lib/camelizeStyleName.js.flow'
Error from chokidar (/data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/node_modules/fbjs/lib): Error: ENOSPC: System limit for number of file watchers reached, watch '/data/deploy_tools/deploy_superset/incubator-superset-0.36.0/superset-frontend/node_modules/fbjs/lib/compactArray.js'^C

Congratulations! Your new Hugo site is created in /Users/jdhao/Blog.
Just a few more steps and you're ready to go:
1. Download a theme into the same-named folder. Choose a theme from https://themes.gohugo.io/, or create your own with the “hugo new theme ” command.
2. Perhaps you want to add some content. You can add single files with “hugo new /.”.
3. Start the built-in live server via “hugo server”.
Visit https://gohugo.io/ for quickstart guide and full documentation.

git init
git submodule add https://github.com/olOwOlo/hugo-theme-even.git themes/even

➜  Blog git:(master) ✗ hugo server -D
Building sites … ERROR 2020/01/04 15:08:40

There are two possible situations that led to this error:
  1. You haven't copied the config.toml yet. See https://github.com/olOwOlo/hugo-theme-even#installation
  2. You have an incompatible update. See https://github.com/olOwOlo/hugo-theme-even/blob/master/CHANGELOG.md#400-2018-11-06

有两种可能的情况会导致这个错误发生:
  1. 你还没有复制 config.toml 参考 https://github.com/olOwOlo/hugo-theme-even/blob/master/README-zh.md#installation
  2. 你进行了一次不兼容的更新 参考 https://github.com/olOwOlo/hugo-theme-even/blob/master/CHANGELOG.md#400-2018-11-06

[permalinks]
  post = "/:year/:month/:day/:filename/"

Front-matter 的区别
hexo的时间格式
date: 2020-01-04 15:00:29
tag: Mac
categories: Mac, Linux
hugo中的时间格式
date: 2020-01-04T15:00:29+08:00
tag: [Mac]。     #[]前面要有空格
categories: [Mac, Linux]    #[]前面要有空格

➜  Blog git:(master) ✗ hugo server

| EN
+------------------+----+
  Pages            | 37
  Paginator pages  |  4
  Non-page files   |  0
  Static files     | 40
  Processed images |  0
  Aliases          |  9
  Sitemaps         |  1
  Cleaned          |  0

Built in 38 ms
Watching for changes in /Users/seven/Blog/{archetypes,content,data,layouts,static,themes}
Watching for config changes in /Users/seven/Blog/config.toml
Environment: "development"
Serving pages from memory
Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender
Web Server is available at http://localhost:1313/ (bind address 127.0.0.1)
Press Ctrl+C to stop

mkdir public
cd public
git init
git remote add upstream https://github.com/jdhao/jdhao.github.io.git

git add .
git commit -m  "hugo commit"
git push -f upstream master # 第一次 push 必须使用 -f，因为此时远程仓库非空，和本地有冲突

!/bin/bash

echo -e "\033[0;32mDeploying updates to GitHub...\033[0m"

# Build the project.
hugo # if using a theme, replace with `hugo -t <YOURTHEME>`

# Go To Public folder
cd public

# Add changes to git.
git add .

# Commit changes.
msg="rebuilding site `date`"
if [ $# -eq 1 ]
  then msg="$1"
fi
git commit -m "$msg"

# Push source and build repos.
git push origin master

# Come Back up to the Project Root
cd ..

➜  Blog git:(master) ✗ hugo server
ERROR 2020/01/04 15:40:34 render of "page" failed: execute of template failed: template: post/single.html:14:5: executing "post/single.html" at <partial "head.html" .>: error calling partial: "/Users/seven/Blog/themes/even/layouts/partials/head.html:22:82": execute of template failed: template: _internal/schema.html:22:82: executing "_internal/schema.html" at <.Params.tags>: range can't iterate over CDH
ERROR 2020/01/04 15:40:34 render of "page" failed: execute of template failed: template: post/single.html:14:5: executing "post/single.html" at <partial "head.html" .>: error calling partial: "/Users/seven/Blog/themes/even/layouts/partials/head.html:22:82": execute of template failed: template: _internal/schema.html:22:82: executing "_internal/schema.html" at <.Params.tags>: range can't iterate over flume
ERROR 2020/01/04 15:40:34 render of "page" failed: execute of template failed: template: post/single.html:14:5: executing "post/single.html" at <partial "head.html" .>: error calling partial: "/Users/seven/Blog/themes/even/layouts/partials/head.html:22:82": execute of template failed: template: _internal/schema.html:22:82: executing "_internal/schema.html" at <.Params.tags>: range can't iterate over 123456
Built in 26 ms
Error: Error building site: failed to render pages: render of "page" failed: execute of template failed: template: post/single.html:14:5: executing "post/single.html" at <partial "head.html" .>: error calling partial: "/Users/seven/Blog/themes/even/layouts/partials/head.html:22:82": execute of template failed: template: _internal/schema.html:22:82: executing "_internal/schema.html" at <.Params.tags>: range can't iterate over CDH,sentry

Sentry，用户身份和权限的映射关系管理 即授权。
CDH的介绍是：Sentry 服务存储身份验证政策元数据并为客户端提供对该元数据的并发安全访问。

<property>
<name>hive.metastore.kerberos.keytab.file</name>
<value>/etc/hive/conf/hive.keytab</value>
</property>

[root@h1 ~]# useradd user1
[root@h1 ~]# useradd user2
[root@h2 ~]# useradd user1
[root@h2 ~]# useradd user2
[root@h3 ~]# useradd user1
[root@h3 ~]# useradd user2
[root@h4 ~]# useradd user1
[root@h4 ~]# useradd user2

sudo kadmin.local -q "addprinc user1"
sudo kadmin.local -q "addprinc user2"

[root@h1 ~]#kinit -kt hive.keytab hive/h1.cat.com
[root@h1 ~]#hive

create database db1;
create database db2;

create table db1.table1 (
name STRING, age STRING
) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';
create table db2.table1 (
name STRING, age STRING
) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';
create table db2.table2 (
name STRING, age STRING
) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';

cat test.csv
leo,19
lilei,23
jack,25
tony,28
eric,18
lucy,16

load data local inpath '/home/user1/test.csv' overwrite into table db1.table1;
load data local inpath '/home/user1/test.csv' overwrite into table db2.table1;
load data local inpath '/home/user1/test.csv' overwrite into table db2.table2;

hive> create role user1_role;
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current user : hive is not allowed to add roles. User has to belong to ADMIN role and have it as current role, for this action.
hive> GRANT ALL ON DATABASE db1 TO ROLE user1_role;
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Permission denied: Principal [name=hive, type=USER] does not have following privileges for operation GRANT_PRIVILEGE [[SELECT with grant, INSERT with grant, UPDATE with grant, DELETE with grant] on Object [type=DATABASE, name=db1]]
hive> GRANT ROLE user1_role TO GROUP user1;
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Invalid principal type in principal Principal [name=user1, type=GROUP]

create role user1_role;
GRANT ALL ON DATABASE db1 TO ROLE user1_role;
GRANT ROLE user1_role TO GROUP user1;

create role user2_role;
GRANT ALL ON DATABASE db2 TO ROLE user2_role;
GRANT ROLE user2_role TO GROUP user2;

FAILED: SemanticException Unable to determine if hdfs://h1.cat.com:8020/user/hive/warehouse/db1.db/table1
is encrypted: org.apache.hadoop.security.AccessControlException: Permission denied: user=user2, access=READ,
inode="/user/hive/warehouse/db1.db/table1":hive:hive:drwxrwx--

(base) [root@h1 ~]#  yum -y install krb5-server krb5-libs krb5-auth-dialog krb5-workstation

(base) [root@h1 ~]#  vim /etc/krb5.conf

# Configuration snippets may be placed in this directory as well
includedir /etc/krb5.conf.d/
[logging]
 default = FILE:/var/log/krb5libs.log
 kdc = FILE:/var/log/krb5kdc.log
 admin_server = FILE:/var/log/kadmind.log
[libdefaults]
 dns_lookup_realm = false
 ticket_lifetime = 24h
 renew_lifetime = 7d
 forwardable = true
 rdns = false
 pkinit_anchors = /etc/pki/tls/certs/ca-bundle.crt
 default_realm = CAT.COM
 #default_ccache_name = KEYRING:persistent:%{uid}
[realms]
 CAT.COM = {
  kdc = master
  admin_server = master
 }
[domain_realm]
.cat.com = CAT.COM
cat.com = CAT.COM

(base) [root@h1 ~]# vim /var/kerberos/krb5kdc/kadm5.acl

*/admin@HADOOP.COM      *
*/master@HADOOP.COM      *

(base) [root@h1 ~]# vim /var/kerberos/krb5kdc/kdc.conf

[kdcdefaults]
 kdc_ports = 88
 kdc_tcp_ports = 88

[realms]
 CAT.COM = {
  #master_key_type = aes256-cts
  #ticket_lifetime = 10m
  max_renewable_life= 7d 0h 0m 0s
  acl_file = /var/kerberos/krb5kdc/kadm5.acl
  dict_file = /usr/share/dict/words
  admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
  supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal camellia256-cts:normal camellia128-cts:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
 }

(base) [root@h1 ~]#  kdb5_util create –r CAT.COM -s

Loading random data

Initializing database '/var/kerberos/krb5kdc/principal' for realm 'CAT.COM',

master key name 'K/M@CAT.COM'

You will be prompted for the database Master Password.

It is important that you NOT FORGET this password.

Enter KDC database master key:

Re-enter KDC database master key to verify:

#此处需要输入Kerberos数据库的密码。

(base) [root@h1 ~]# kadmin.local

Authenticating as principal root/admin@CAT.COM with password.

kadmin.local:  addprinc admin/admin@CAT.COM

WARNING: no policy specified for admin/admin@CAT.COM; defaulting to no policy

Enter password for principal "admin/admin@CAT.COM":

Re-enter password for principal "admin/admin@CAT.COM":

Principal "admin/admin@CAT.COM" created.

kadmin.local:  exit

[root@h1 ~]# systemctl enable kadmin
Created symlink from /etc/systemd/system/multi-user.target.wants/kadmin.service to /usr/lib/systemd/system/kadmin.service.
[root@h1 ~]# systemctl start krb5kdc
[root@h1 ~]# systemctl start kadmin

[root@h1 ~]# systemctl enable krb5kdc

[root@h2 ~]#yum -y install krb5-libs krb5-workstation
[root@h3 ~]#yum -y install krb5-libs krb5-workstation
[root@h4 ~]#yum -y install krb5-libs krb5-workstation

[root@h1 ~]# yum -y install openldap-clients

scp /etc/krb5.conf root@h2:/etc/

(base) [root@h2 ~]# kadmin.local

Authenticating as principal admin/admin@CAT.COM with password.

kadmin.local:  addprinc cloudera-scm/admin@CAT.COM

WARNING: no policy specified for cloudera-scm/admin@CAT.COM; defaulting to no policy

Enter password for principal "cloudera-scm/admin@CAT.COM":

Re-enter password for principal "cloudera-scm/admin@CAT.COM":

Principal "cloudera-scm/admin@CAT.COM" created.

kadmin.local:  exit

(base) [root@h1 ~]# kadmin.local

Authenticating as principal admin/admin@CAT.COM with password.

kadmin.local:  addprinc user1@CAT.COM

WARNING: no policy specified for user1@CAT.COM; defaulting to no policy

Enter password for principal "user1@CATCOM":

Re-enter password for principal "user1@CAT.COM":

Principal "user1@CAT.COM" created.

kadmin.local:  exit

[root@h1 ~]# kdestroy
[root@h1 ~]# kinit user1
#设置user1登陆密码
Password for user1@CAT.COM:
#查看user1的有效期
(base) [root@h1 ~]# klist
Ticket cache: FILE:/tmp/krb5cc_0
Default principal: user1@CAT.COM
Valid starting       Expires              Service principal
12/26/2019 18:29:17  12/27/2019 18:29:17  krbtgt/CAT.COM@CAT.COM
        renew until 01/03/2019 18:29:17

[root@h1 ~]#useradd user1
[root@h2 ~]#useradd user1
[root@h3 ~]#useradd user1

[root@h1 ~]# usermod -G hdfs,hadoop user1
[root@h1 ~]# usermod -G hadoop user1

#先用user1登陆kerberos
[root@h1 conf]# su - user1
[root@h1 conf]# kinit user1
Password for user1@CAT.COM:
[root@h1 conf]# klist
Ticket cache: FILE:/tmp/krb5cc_0
Default principal: user1@CAT.COM
Valid starting       Expires              Service principal
12/30/2019 22:27:40  12/31/2019 22:27:40  krbtgt/CAT.COM@CAT.COM
        renew until 01/06/2020 22:27:40
#测试mr demo
[root@h1 conf]# hadoop jar /opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 10 1
WARNING: Use "yarn jar" to launch YARN applications.
Number of Maps  = 10
Samples per Map = 1
Wrote input for Map #0
Wrote input for Map #1
Wrote input for Map #2
Wrote input for Map #3
Wrote input for Map #4
Wrote input for Map #5
Wrote input for Map #6
Wrote input for Map #7
Wrote input for Map #8
Wrote input for Map #9
Starting Job
19/12/30 22:32:49 INFO client.RMProxy: Connecting to ResourceManager at h1.cat.com/192.168.1.171:8032
19/12/30 22:32:50 INFO hdfs.DFSClient: Created token for user2: HDFS_DELEGATION_TOKEN owner=user2@CAT.COM, renewer=yarn, realUser=, issueDate=1577716370019, maxDate=1578321170019, sequenceNumber=4, masterKeyId=10 on 192.168.1.171:8020
19/12/30 22:32:50 INFO security.TokenCache: Got dt for hdfs://h1.cat.com:8020; Kind: HDFS_DELEGATION_TOKEN, Service: 192.168.1.171:8020, Ident: (token for user2: HDFS_DELEGATION_TOKEN owner=user2@CAT.COM, renewer=yarn, realUser=, issueDate=1577716370019, maxDate=1578321170019, sequenceNumber=4, masterKeyId=10)
19/12/30 22:32:50 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /user/user2/.staging/job_1577701614246_0003
19/12/30 22:32:50 INFO input.FileInputFormat: Total input files to process : 10
19/12/30 22:32:50 INFO mapreduce.JobSubmitter: number of splits:10
19/12/30 22:32:50 INFO Configuration.deprecation: yarn.resourcemanager.system-metrics-publisher.enabled is deprecated. Instead, use yarn.system-metrics-publisher.enabled
19/12/30 22:32:50 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1577701614246_0003
19/12/30 22:32:50 INFO mapreduce.JobSubmitter: Executing with tokens: [Kind: HDFS_DELEGATION_TOKEN, Service: 192.168.1.171:8020, Ident: (token for user2: HDFS_DELEGATION_TOKEN owner=user2@CAT.COM, renewer=yarn, realUser=, issueDate=1577716370019, maxDate=1578321170019, sequenceNumber=4, masterKeyId=10)]
19/12/30 22:32:51 INFO conf.Configuration: resource-types.xml not found
19/12/30 22:32:51 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.
19/12/30 22:32:51 INFO impl.YarnClientImpl: Submitted application application_1577701614246_0003
19/12/30 22:32:51 INFO mapreduce.Job: The url to track the job: http://h1.cat.com:8088/proxy/application_1577701614246_0003/
19/12/30 22:32:51 INFO mapreduce.Job: Running job: job_1577701614246_0003
19/12/30 22:33:04 INFO mapreduce.Job: Job job_1577701614246_0003 running in uber mode : false
19/12/30 22:33:04 INFO mapreduce.Job:  map 0% reduce 0%
19/12/30 22:33:14 INFO mapreduce.Job:  map 20% reduce 0%
19/12/30 22:33:19 INFO mapreduce.Job:  map 30% reduce 0%
19/12/30 22:33:20 INFO mapreduce.Job:  map 40% reduce 0%
19/12/30 22:33:24 INFO mapreduce.Job:  map 50% reduce 0%
19/12/30 22:33:25 INFO mapreduce.Job:  map 60% reduce 0%
19/12/30 22:33:30 INFO mapreduce.Job:  map 80% reduce 0%
19/12/30 22:33:35 INFO mapreduce.Job:  map 100% reduce 0%
19/12/30 22:33:41 INFO mapreduce.Job:  map 100% reduce 100%
19/12/30 22:33:42 INFO mapreduce.Job: Job job_1577701614246_0003 completed successfully
19/12/30 22:33:42 INFO mapreduce.Job: Counters: 54
        File System Counters
                FILE: Number of bytes read=49
                FILE: Number of bytes written=2433690
                FILE: Number of read operations=0
                FILE: Number of large read operations=0
                FILE: Number of write operations=0
......................
Job Finished in 53.382 seconds
Estimated value of Pi is 3.60000000000000000000

beeline> !connect jdbc:hive2://localhost:10000/;principal=hive/h1@CAT.COM
Connecting to jdbc:hive2://localhost:10000/;principal=hive/h1@CAT.COM
Connected to: Apache Hive (version 2.1.1-cdh6.3.1)
Driver: Hive JDBC (version 2.1.1-cdh6.3.1)
Transaction isolation: TRANSACTION_REPEATABLE_READ
0: jdbc:hive2://localhost:10000/>

[root@h1 ~]# hive
WARNING: Use "yarn jar" to launch YARN applications.
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/opt/cloudera/parcels/CDH-6.3.1-1.cdh6.3.1.p0.1470567/jars/log4j-slf4j-impl-2.8.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/opt/cloudera/parcels/CDH-6.3.1-1.cdh6.3.1.p0.1470567/jars/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]

Logging initialized using configuration in jar:file:/opt/cloudera/parcels/CDH-6.3.1-1.cdh6.3.1.p0.1470567/jars/hive-common-2.1.1-cdh6.3.1.jar!/hive-log4j2.properties Async: false
Exception in thread "main" java.lang.RuntimeException: java.io.IOException: Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]; Host Details : local host is: "h1.cat.com/192.168.1.171"; destination host is: "h1.cat.com":8020;
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:604)
        at org.apache.hadoop.hive.ql.session.SessionState.beginStart(SessionState.java:545)
        at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:763)
        at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:699)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.util.RunJar.run(RunJar.java:313)
        at org.apache.hadoop.util.RunJar.main(RunJar.java:227)
Caused by: java.io.IOException: Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]; Host Details : local host is: "h1.cat.com/192.168.1.171"; destination host is: "h1.cat.com":8020;
        at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:808)
        at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1503)
        at org.apache.hadoop.ipc.Client.call(Client.java:1445)
        at org.apache.hadoop.ipc.Client.call(Client.java:1355)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
        at com.sun.proxy.$Proxy28.getFileInfo(Unknown Source)
        at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:875)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
        at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
        at com.sun.proxy.$Proxy29.getFileInfo(Unknown Source)
        at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1630)
        at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1496)
        at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1493)
        at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
        at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1508)
        at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1617)
        at org.apache.hadoop.hive.ql.session.SessionState.createRootHDFSDir(SessionState.java:712)
        at org.apache.hadoop.hive.ql.session.SessionState.createSessionDirs(SessionState.java:650)
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:580)
        ... 9 more
Caused by: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]
        at org.apache.hadoop.ipc.Client$Connection$1.run(Client.java:756)

cd /var/kerberos/krb5kdc/

kadmin.local -q "addprinc -randkey hive/h1.cat.com@CAT.COM "
kadmin.local -q "addprinc -randkey hive/h2.cat.com@CAT.COM "
kadmin.local -q "addprinc -randkey hive/h3.cat.com@CAT.COM "

kadmin.local -q "xst  -k hive.keytab  hive/h1.cat.com@CAT.COM "
kadmin.local -q "xst  -k hive.keytab  hive/h2.cat.com@CAT.COM "
kadmin.local -q "xst  -k hive.keytab  hive/h3.cat.com@CAT.COM "

$ scp hive.keytab cdh1:/etc/hive/conf
$ scp hive.keytab cdh2:/etc/hive/conf
$ scp hive.keytab cdh3:/etc/hive/conf

$ ssh cdh1 "cd /etc/hive/conf/;chown hive:hadoop hive.keytab ;chmod 400 *.keytab"
$ ssh cdh2 "cd /etc/hive/conf/;chown hive:hadoop hive.keytab ;chmod 400 *.keytab"
$ ssh cdh3 "cd /etc/hive/conf/;chown hive:hadoop hive.keytab ;chmod 400 *.keytab"

[user1@h1 ~]$ kinit -kt hive.keytab hive/h1
[user1@h1 ~]$ hive

beeline> !connect jdbc:hive2://h1:10000/;principal=hive/h1@CAT.COM
Connecting to jdbc:hive2://h1:10000/;principal=hive/h1@CAT.COM
19/12/30 17:36:50 [main]: WARN jdbc.HiveConnection: Failed to connect to h1:10000
Could not open connection to the HS2 server. Please check the server URI and if the URI is correct, then ask the administrator to check the server status.
Error: Could not open client transport with JDBC Uri: jdbc:hive2://h1:10000/;principal=hive/h1@CAT.COM: java.net.ConnectException: Connection refused (Connection refused) (state=08S01,code=0)

beeline> !connect jdbc:hive2://h1:10000/;principal=hive/h1@CAT.COM
Connecting to jdbc:hive2://h1:10000/;principal=hive/h1@CAT.COM
19/12/30 19:10:22 [main]: ERROR transport.TSaslTransport: SASL negotiation failure
javax.security.sasl.SaslException: GSS initiate failed

INFO kt_renewer
Renewing kerberos ticket to work around kerberos 1.8.1: /bin/kinit -R -c /var/run/hue/hue_krb5_ccache
ERROR kt_renewer Couldn't renew kerberos ticket in order to work around Kerberos 1.8.1 issue. Please check that the ticket for 'hue/$server1@ANYTHING.COM' is still renewable:
$ klist -f -c /var/run/hue/hue_krb5_ccache
If the 'renew until' date is the same as the 'valid starting' date, the ticket cannot be renewed. Please check your KDC configuration, and the ticket renewal policy (maxrenewlife) for the 'hue/$server1@ANYTHING.COM' and `krbtgt' principals.
[19/Feb/2019 07:32:04 ] settings INFO Welcome to Hue 3.9.0

[kdcdefaults]
 kdc_ports = 88
 kdc_tcp_ports = 88

[realms]
 CAT.COM = {
  #master_key_type = aes256-cts
  #ticket_lifetime = 10m
  max_renewable_life= 7d 0h 0m 0s
  acl_file = /var/kerberos/krb5kdc/kadm5.acl
  dict_file = /usr/share/dict/words
  admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
  supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal camellia256-cts:normal ca
mellia128-cts:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
 }

================================================================

### 其他总结

**给新建用户分配权限步骤**

- 1.创建kerberos用户和密码

**2. 在所有机器节点添加 user1 用户**

```bash
useradd user1        # 每台节点都执行

http://<server-ip>:7180     # 默认账号/密码：admin / admin

tar -xzvf apache-flume-1.8.0-src.tar.gz

cd apache-flume-1.8.0-src\flume-ng-sdk\src\main\thrift

thrift --gen py flume.thrit

#!/usr/bin/env python
#encoding: utf-8
from genpy.flume import ThriftSourceProtocol  
from genpy.flume.ttypes import ThriftFlumeEvent  
from thrift.transport import TTransport, TSocket  
from thrift.protocol import TCompactProtocol

class _Transport(object):  
    def __init__(self, thrift_host, thrift_port, timeout=None, unix_socket=None):  
        self.thrift_host = thrift_host  
        self.thrift_port = thrift_port  
        self.timeout = timeout  
        self.unix_socket = unix_socket 
        self._socket = TSocket.TSocket(self.thrift_host, self.thrift_port, self.unix_socket)  
        self._transport_factory = TTransport.TFramedTransportFactory()  
        self._transport = self._transport_factory.getTransport(self._socket)

def connect(self):  
        try:  
            if self.timeout:  
                self._socket.setTimeout(self.timeout)  
            if not self.is_open():  
                self._transport = self._transport_factory.getTransport(self._socket)  
                self._transport.open()  
        except Exception, e:
            print '#'*20  
            print(e)  
            self.close()

def is_open(self):  
        return self._transport.isOpen()

def get_transport(self):  
        return self._transport

def close(self):  
        self._transport.close()

class FlumeClient(object):  
    def __init__(self, thrift_host, thrift_port, timeout=None, unix_socket=None):  
        self._transObj = _Transport(thrift_host, thrift_port, timeout=timeout, unix_socket=unix_socket)  
        self._protocol = TCompactProtocol.TCompactProtocol(trans=self._transObj.get_transport())  
        self.client = ThriftSourceProtocol.Client(iprot=self._protocol, oprot=self._protocol)  
        self._transObj.connect()

def send(self, event):  
        try:  
            self.client.append(event)  
        except Exception, e:  
            print(e)  
        finally:  
            self._transObj.connect()

def send_batch(self, events):  
        try:  
            self.client.appendBatch(events)  
        except Exception, e:  
            print(e)  
        finally:  
            self._transObj.connect()

def close(self):  
        self._transObj.close()

if __name__ == '__main__':  
    import random  
    flume_client = FlumeClient('xxxxxx',4444)  
    events = [ThriftFlumeEvent({'GAMENAME':'NYHX'},'NYevents under hello world %s' %(random.randint(0, 100))) for _ in range(100)]    
    events1 = [ThriftFlumeEvent({'GAMENAME':'FSZHS'},'FSevents under hello world %s' %(random.randint(0, 100))) for _ in range(100)]  
    flume_client.send_batch(events)  
    flume_client.send_batch(events1)  
    flume_client.close() 

tier1.sources.source1.selector.type = multiplexing
tier1.sources.source1.selector.header = GAMENAME
tier1.sources.source1.selector.mapping.FSZHS = channel1

# 配置kafka接收数据
#tier1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSink
#tier1.sinks.sink1.brokerList = localhost:9092
#tier1.sinks.sink1.topic = test
#tier1.sinks.sink1.serializer.class = kafka.serializer.StringEncoder
#tier1.sinks.sink1.channel = channel1

#!/usr/bin/env python
# encoding: utf-8
from multiprocessing.dummy import Pool as ThreadPool
import time,datetime,os,sys

from flume import FlumeClient
from genpy.flume.ttypes import ThriftFlumeEvent

hour=datetime.datetime.now().strftime('%H')

if hour == '23':
    day=(datetime.datetime.now() + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
else:
    day=datetime.datetime.now().strftime('%Y-%m-%d')

def read_server():
    time.sleep(5)
    while True:
        try:
            if not os.path.exists('/data/scribe/log_xxx/PRODUCT_xxx_ANDR_%s/PRODUCT_xxx_ANDR_%s_00000'%(day,day)):
                print('服务端日志文件不存在,sleep........50')
                time.sleep(50)
                continue
            fread_tell = '0'
            if os.path.exists('/home/eric/servertell/server_tell_%s.txt' % day):
                fread = open('/home/eric/servertell/server_tell_%s.txt' % day)
                fread_tells = fread.readlines()
                fread_tell = fread_tells[-1]
                fread.close()
            else:
                fread = open('/home/eric/servertell/server_tell_%s.txt' % day,'a+')
                fread.write('0\n')
                fread.close()
            ffile = open('/data/scribe/log_xxx/PRODUCT_xxxx_ANDR_%s/PRODUCT_xxx_ANDR_%s_00000'%(day,day))
            if not len(fread_tell):
                continue
            ffile.seek(int(fread_tell))
            lines = ffile.readlines()
            log = None
            print'未读行数======',(len(lines))
            if len(lines)<1:
                time.sleep(10)
                print '行数少于1休眠10s..................'
            else:
                for line in lines:
                    if line == line.strip() or len(line)<10:
                        time.sleep(5)
                        print "空行或者小于10行.........."
                        continue
                    else:
                        print "reading................."
                        print line
                        flume_client = FlumeClient('xxxxxxx',4444)
                        events1 = [ThriftFlumeEvent({'GAMENAME':'FSZHS'},line.strip('\n'))]
                        flume_client.send_batch(events1)
                        #bilog.sendBI(line.strip('\n'),'PRODUCT_CKQY_ANDR_%s'%(day))
            where1 = ffile.tell()
            while True:
                where1 = where1 -1
                seek = ffile.seek(where1,os.SEEK_SET)
                if ffile.read(1)=='\n':
                    break
            print '读取位置',(where1)
            f = open('/home/eric/servertell/server_tell_%s.txt' % day,'a+')
            f.write('\n')
            f.write(str(where1))
            f.close()
            ffile.close()
            if log:
                log.close()
        except Exception as e:
            traceback.print_exc()
            time.sleep(5)
            continue
if __name__ == '__main__':
    pool = ThreadPool(processes = 1)
    pool.apply_async(read_server)
    pool.close()
    pool.join()

graph LR
A[scribe] --> B[disk] 
A --> C(hdfs)
B --> D[python ThriftFlumeEvent发送到flume]
D --> F[Flume]
F --sink--> G[kafka]
G--scala--> H[flink]

Ubuntu 16.04
conda环境下的python 3.7

jdk1.8.0_202，scala 2.11.6 ，zookeeper-3.4.5，kafka_2.11-2.3.0，flume-1.8.0，flink-1.8.0

export JAVA_HOME=/opt/jdk1.8.0_202  ## 这里要注意目录要换成自己解压的jdk 目录
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH

wget https://downloads.lightbend.com/scala/2.11.6/scala-2.11.6.tgz

tar -zxvf scala-2.11.6.tgz -C /opt/scala-2.11.6

vim ~/.bashrc

source ~/.bashrc

添加环境变量如下
export SCALA_HOME=/opt/scala-2.11.6
export PATH=$PATH:${SCALA_HOME}/bin

验证 scala -version

下载 wget http://archive.apache.org/dist/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz

解压 tar -zxvf zookeeper-3.4.5.tar.gz -C /opt/zookeeper-3.4.5

进入解压目录  cd /opt/zookeeper-3.4.5/conf/

修改zookeeper数据存放地址，dataDir,dataLogDir

dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/log

mkdir /data/zookeeper/data
mkdir /data/zookeeper/log
在/data/zookeeper/data 目录新建一个myid文件，内容为1，代表服务器的编号是1

export ZOOKEEPER_HOME=/opt/zookeeper-3.4.5
export PATH=.:$ZOOKEEPER_HOME/bin:$JAVA_HOME/bin:$PATH

vi ~/.bash_profile

export KAFKA_HOME=/opt/kafka_2.11-2.3.0

export PATH=$KAFKA_HOME/bin:$PATH

cd /opt/kafka_2.11-2.3.0/
bin/zookeeper-server-start.sh config/zookeeper.properties

//配置了环境变量,1代表标准输出,2代表错误输出,把2重定向到标准输出1需要使用&,最后的&表示后台运行
nohup bin/kafka-server-start.sh config/server.properties > kafka_start.log 2>&1 & 

//启动kafka
bin/kafka-server-start.sh config/server.properties 
//停止kafka
bin/kafka-server-stop.sh
//创建topic,--replication-factor指定副本个数,--partitions指定分区个数
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
//查看所有的topic信息
bin/kafka-topics.sh --list --zookeeper localhost:2181
//启动生产者
bin/kafka-console-producer.sh --broker-list your.host.name:9092 --topic test
//启动消费者
bin/kafka-console-consumer.sh --bootstrap-server your.host.name:9092 --topic test --from-beginning
//删除topic
bin/kafka-topics.sh --delete --zookeeper localhost:2181 --topic test

//用于区分broker,必须唯一
broker.id=0

//代理将向生产者和消费者宣传的主机名和端口。 如果没有设定，它将使用“listeners”的值。否则，它从java.net.InetAddress.getCanonicalHostName获得。
advertised.listeners=PLAINTEXT://外网ip:9092

//配置zookeeper的地址,zookeeper集群地址以逗号隔开
zookeeper.connect=localhost:2181

//配置数据存放目录
log.dirs=/data/kafka2.3

#listeners=PLAINTEXT://:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
zookeeper.connect=localhost:2181

#listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://外网ip:9092
zookeeper.connect=localhost:2181

wget http://archive.apache.org/dist/flume/1.8.0/apache-flume-1.8.0-bin.tar.gz
tar -zxvf apache-flume-1.8.0-bin.tar.gz
重命名 mv apache-flume-1.8.0-bin flume-1.8.0

vi ~/.bash_profile
export FLUME_HOME=/opt/flume-1.8.0
export PATH=$PATH:$FLUME_HOME/bin

#example.conf: A single-node Flume configuration

#Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

#Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444

#Describe the sink
#打印内容到日志中
a1.sinks.k1.type = logger

#Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

#Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

tier1.sources  = source1
tier1.channels = channel1
tier1.sinks    = sink1

tier1.sources.source1.channels = channel1
tier1.channels.channel1.type   = memory
tier1.sources.source1.type = thrift
tier1.sources.source1.bind = 0.0.0.0
tier1.sources.source1.port = 4444

tier1.sources.source1.selector.type = multiplexing
tier1.sources.source1.selector.header = GAMENAME
tier1.sources.source1.selector.mapping.FSZHS = channel1

#配置sink下沉到kafka，kafka来消费数据
tier1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSink
tier1.sinks.sink1.brokerList = localhost:9092
tier1.sinks.sink1.topic = test
tier1.sinks.sink1.serializer.class = kafka.serializer.StringEncoder
tier1.sinks.sink1.channel = channel1

tier1.channels.channel1.capacity = 5000
tier1.channels.channel1.transactionCapacity = 2000

#日志中打印出内容
#tier1.sinks.sink1.type = logger
#tier1.sinks.sink1.channel = channel1

wget https://archive.apache.org/dist/flink/flink-1.8.0/flink-1.8.0-bin-scala_2.11.tgz

解压 tar -zxvf apache-flume-1.8.0-bin.tar.gz

pip install kafka-python==1.3.1
pip3 uninstall kafka
pip uninstall kafka
pip uninstall kafka-python
conda install -c conda-forge kafka-python==1.3.1