log_format main '{"ip":"$remote_addr","request_method":"$request_method","request_uri":"$document_uri","params":$fmt_params,"browser":"$http_user_agent","person_id":"$fmt_person_id","identity_id":"$fmt_identity_id","class_id":"$fmt_class_id","bureau_id":"$fmt_bureau_id","request_time":"$fmt_localtime","location":$fmt_location,"http_status":"$status","area_code":"$fmt_area_code"}';

    access_log  logs/access.log  main;

        map $host $fmt_localtime {
                        default '';
        }
        map $host $fmt_person_id {
                        default '';
        }
        map $host $fmt_identity_id {
                        default '';
        }
        map $host $fmt_class_id {
                        default '';
        }
        map $host $fmt_bureau_id {
                        default '';
        }
        map $host $fmt_params {
                        default '{}';
        }
        map $host $fmt_location {
                        default '';
        }
        map $host $fmt_area_code {
                        default '';
        }

        log_by_lua_block {
                local cjson = require "cjson"

                ngx.var.fmt_localtime = ngx.localtime();

                ngx.var.fmt_area_code = v_area_code


                local _person_id = ""
                local cookie_person_id = ngx.var.cookie_person_id
                if cookie_person_id ~= nil then
                                _person_id = tostring(cookie_person_id)
                end
                if #ngx.var.fmt_person_id == 0 then
                                ngx.var.fmt_person_id = _person_id
                end


                local _identity_id = ""
                local cookie_identity_id = ngx.var.cookie_identity_id
                if cookie_identity_id ~= nil then
                                _identity_id = tostring(cookie_identity_id)
                end
                if #ngx.var.fmt_identity_id == 0 then
                                ngx.var.fmt_identity_id = _identity_id
                end


                local _class_id = ""
                local cookie_class_id = ngx.var.cookie_class_id
                if cookie_class_id ~= nil then
                                _class_id = string.gsub(tostring(cookie_class_id), "\"", "")
                end
                ngx.var.fmt_class_id = _class_id


                local _bureau_id = ""
                local cookie_background_bureau_id = ngx.var.cookie_background_bureau_id
                if cookie_background_bureau_id ~= nil then
                                _bureau_id = tostring(cookie_background_bureau_id)
                end
                local cookie_bureau_id = ngx.var.cookie_bureau_id
                if cookie_bureau_id ~= nil then
                                _bureau_id = tostring(cookie_bureau_id)
                end
                ngx.var.fmt_bureau_id = _bureau_id


                ngx.var.fmt_params = "{}"
                -- 遍历数组
                function IsInTable(value, tbl)
                                for k,v in ipairs(tbl) do
                                                if v == value then
                                                                return true;
                                                end
                                end
                                return false;
                end
                --解码
                function decodeURI(s)
                                s = string.gsub(s, '%%(%x%x)', function(h) return string.char(tonumber(h, 16)) end)
                                return s
                end

                local _params = ""
                local get_args = ngx.var.args
                if get_args ~= nil then
                                _params = get_args
                end
                local post_args = ngx.var.request_body
                if post_args ~= nil then
                                _params = post_args
                end
                if string.len(_params) ~= 0 then
                                local flag = 0
                                local tab = {}
                                for _k,_tab in pairs(v_uri_whitelist) do
                                                local uri_name = tostring(_tab["uri_name"])
                                                local _args = _tab["args"]
                                                if uri_name == tostring(ngx.var.document_uri) then
                                                                local argsb = Split(_params,"&")
                                                                for i=1,#argsb do
                                                                                local argss = Split(argsb[i],"=")
                                                                                if IsInTable(tostring(argss[1]),_args) == true then
                                                                                                tab[tostring(argss[1])] = ngx.encode_base64(ngx.encode_base64(decodeURI(tostring(argss[2]))))
                                                                                end
                                                                end
                                                                flag = 1
                                                end
                                end
                                if flag == 1 then
                                                ngx.var.fmt_params = tostring(cjson.encode(tab))
                                end
                end


                local _ip = ngx.var.remote_addr
                if string.len(_ip) ~= 0 then
                                local _tab = {}
                                local city = require "datx.city"
                                local c = city:new("/usr/local/openresty/nginx/conf/17monipdb.datx")
                                local data = c:find(_ip)
                                _tab["country"] = ngx.encode_base64(ngx.encode_base64(tostring(data[1])))
                                _tab["province"] = ngx.encode_base64(ngx.encode_base64(tostring(data[2])))
                                _tab["city"] = ngx.encode_base64(ngx.encode_base64(tostring(data[3])))
                                ngx.var.fmt_location = tostring(cjson.encode(_tab))
                else
                                ngx.var.fmt_location = "{}"
                end

        }

        body_filter_by_lua '
                        if tostring(ngx.var.document_uri) == "/dsideal_yy/caslogin/casLoginWithShare" then
                                        if #tostring(ngx.arg[1]) ~= 0 then
                                                        local cjson = require "cjson"
                                                        local jsonTab = cjson.decode(tostring(ngx.arg[1]))
                                                        if tostring(jsonTab["success"]) == "true" then
                                                                        ngx.var.fmt_person_id = tostring(jsonTab["person_id"])
                                                                        ngx.var.fmt_identity_id = tostring(jsonTab["identity_id"])
                                                        end
                                        end
                        end
    ';





location ^~ /bigdata
        {
            proxy_pass http://data.edusoa.com/bigdata/;
        }
        location ^~ /BigDataWeb
        {
            proxy_pass http://data.edusoa.com:8899/BigDataWeb;
        }






v_area_code = "daxinganling"

v_uri_whitelist = {
        {
                uri_name = "/dsideal_yy/ypt/blog/saveArticle",
                args = {"title","overview"}
        },
        {
                uri_name = "/dsideal_yy/ypt/blog/updateArticle",
                args = {"title","overview"}
        },
        {
                uri_name = "/dsideal_yy/ypt/uploadres/oss_upload_resource",
                args = {"file_name"}
        }
}








filebeat.prospectors:
- input_type: log
  paths:
    - /usr/local/openresty/nginx/logs/access.log
output.kafka:
  hosts: ["114.115.171.119:9092"]
  topic: need_process_nginx_access_log
  partition.round_robin:
    reachable_only: false
  required_acks: 1

 

 

http://dsideal-yy.oss-cn-qingdao.aliyuncs.com/datx.zip