Daily 0503

FROM python:2.7

WORKDIR /
RUN git clone https://github.com/Yelp/elastalert.git
RUN cd  elastalert && git checkout v0.1.30 && pip install -r requirements.txt


WORKDIR /elastalert/
RUN ls -l 

RUN apt-get install -y tzdata
ENV TZ Asia/Shanghai

CMD ["python", "elastalert/elastalert.py"]

GET /my_index/_search
{
  "query": {
    "match_phrase": {
      "names": "Lincoln Smith"
    }
  }
}

GET /_analyze
{
  "analyzer": "standard",
  "text": [
    "John Abraham",
    "Lincoln Smith"
  ]
}

GET /my_index/_doc/9 

PUT /my_index/_doc/9  
{
  "names": ["John Abraham", "Lincoln Smith"]
}


GET /my_blog/_search
{
  "query": {
    "match_phrase": {
      "body": "quick brown fox"
    }
  }
}

GET /my_blog/_search
{
  "query": {
    "dis_max": {
      "queries": [
        {
          "match": {
            "title": "Quick pets"
          }
        },
        {
          "match": {
            "body": "Quick pets"
          }
        }
      ],
      "tie_breaker": 0.7
    }
  }
}



GET /my_blog/_search
{
  "query": {
    "dis_max": {
      "queries": [
        {
          "match": {
            "title": "Brown fox"
          }
        },
        {
          "match": {
            "body": "Brown fox"
          }
        }
      ]
    }
  }
}




# 返回了并不是用户想要的结果 
GET /my_blog/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "match": {
            "title": "Brown fox"
          }
        },
        {
          "match": {
            "body": "Brown fox"
          }
        }
      ]
    }
  }
}



PUT /my_blog/_doc/2
{
  "title": "Keeping pets healthy",    
  "body":  "My quick brown fox eats rabbits on a regular basis."
}

PUT /my_blog/_doc/1
{
  "title": "Quick brown rabbits",    
  "body":  "Brown rabbits are commonly seen."
}


PUT /my_blog



GET /my_index/_search
{
  "query": {
    "match": {
      "title": {
        "query": "BROWN DOG",
        "operator": "and"
      }
    }
  }
}

GET /my_index/_search 
{
  "query": {
    "match": {
      "title": "QUICK!"
    }
  }
}


POST /my_index/_bulk
{ "index": { "_id": 1 }}
{ "title": "The quick brown fox" }
{ "index": { "_id": 2 }} 
{ "title": "The quick brown fox jumps over the lazy dog" }
{ "index": { "_id": 3 }}
{ "title": "The quick brown fox jumps over the quick dog" } 
{ "index": { "_id": 4 }} 
{ "title": "Brown fox brown dog" }


# 为什么 shard 为 1 
PUT /my_index
{
  "settings": {
    "number_of_shards": 1
  }
  
}


DELETE /my_index 


GET /my_store/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "term": {
            "productID": "KDKE-B-9947-#kL5"
          }
        },
        {
          "bool": {
            "must": [
              {
                "term": {
                  "price": "30"
                }
              },
              {
                "term": {
                  "productID": {
                    "value": "JODL-X-1937-#pV7"
                  }
                }
              }
            ]
          }
        }
      ]
    }
  }
}


DELETE /my_store

GET /my_store/_search
{
  "query": {
    "bool": {
      "must": [
        {
        "match_all": {}
        }
      ], 
      "filter": [
        {
          "term": {
            "productID": "XHDK-A-1293-#fJ3"
          }
        }
      ]
    }
  }
}


PUT /my_store 
{     
  "mappings" : {         
      "properties" : {                 
        "productID" : {                     
          "type" : "keyword"
        }            
        }        
      }     
    }


POST /my_store/_doc/_bulk 
{ "index": { "_id": 1 }} 
{ "price" : 10, "productID" : "XHDK-A-1293-#fJ3" } 
{ "index": { "_id": 2 }} 
{ "price" : 20, "productID" : "KDKE-B-9947-#kL5" } 
{ "index": { "_id": 3 }} 
{ "price" : 30, "productID" : "JODL-X-1937-#pV7" } 
{ "index": { "_id": 4 }} 
{ "price" : 30, "productID" : "QQPX-R-3956-#aD8" }



GET /us-tweet/_search
{ 
  "query": {
    "match_all": {}
  }
  , "_source": ["tweet", "date"]
}

GET /us-tweet

GET /us-tweet/_search?search_type=dfs_query_then_fetch

# explain API to understand why one particular document matched or, more important, why it didn't match .
GET /us-tweet/_doc/12/_explain 
{
  "query": {
    "bool": {
      "filter": {
        "term": {
          "user_id": "3"
        }
      },
      "must": 
        {
          "match": {
            "tweet": "honeymoon"
          }
        }
      
    }
  }
}

GET /_search?explain=true
{
  "query": {
    "match": {
      "tweet": "honeymoon"
    }
  }
}

GET /_search
{
  "query": {
    "bool": {
      "must": { "match": {
        "tweet": "managee text search"
        } 
      },
      "filter": [
        { "term": {
          "user_id": "2"
          }
        }
      ]
    }
  }
}



GET /us-tweet/_search
{
  "query": {
    "bool": {
      "filter": {
        "term": {
          "user_id": 1
        }
      }
    }
  },
  "sort": {
    "date": {
      "order": "desc"
    }
  }
}


GET /_search

PUT /us-user
PUT /gb-user
PUT /gb-tweet
PUT /us-tweet

POST /_bulk
{"create":{"_index":"us-user","_id":"1"}}
{"email":"john@smith.com","name":"John Smith","username":"@john"}
{"create":{"_index":"gb-user","_id":"2"}}
{"email":"mary@jones.com","name":"Mary Jones","username":"@mary"}
{"create":{"_index":"gb-tweet","_id":"3"}}
{"date":"2014-09-13","name":"Mary Jones","tweet":"Elasticsearch means full text search has never been so easy","user_id":2}
{"create":{"_index":"us-tweet","_id":"4"}}
{"date":"2014-09-14","name":"John Smith","tweet":"@mary it is not just text, it does everything","user_id":1}
{"create":{"_index":"gb-tweet","_id":"5"}}
{"date":"2014-09-15","name":"Mary Jones","tweet":"However did I manage before Elasticsearch?","user_id":2}
{"create":{"_index":"us-tweet","_id":"6"}}
{"date":"2014-09-16","name":"John Smith","tweet":"The Elasticsearch API is really easy to use","user_id":1}
{"create":{"_index":"gb-tweet","_id":"7"}}
{"date":"2014-09-17","name":"Mary Jones","tweet":"The Query DSL is really powerful and flexible","user_id":2}
{"create":{"_index":"us-tweet","_id":"8"}}
{"date":"2014-09-18","name":"John Smith","user_id":1}
{"create":{"_index":"gb-tweet","_id":"9"}}
{"date":"2014-09-19","name":"Mary Jones","tweet":"Geo-location aggregations are really cool","user_id":2}
{"create":{"_index":"us-tweet","_id":"10"}}
{"date":"2014-09-20","name":"John Smith","tweet":"Elasticsearch surely is one of the hottest new NoSQL products","user_id":1}
{"create":{"_index":"gb-tweet","_id":"11"}}
{"date":"2014-09-21","name":"Mary Jones","tweet":"Elasticsearch is built for the cloud, easy to scale","user_id":2}
{"create":{"_index":"us-tweet","_id":"12"}}
{"date":"2014-09-22","name":"John Smith","tweet":"Elasticsearch and I have left the honeymoon stage, and I still love her.","user_id":1}
{"create":{"_index":"gb-tweet","_id":"13"}}
{"date":"2014-09-23","name":"Mary Jones","tweet":"So yes, I am an Elasticsearch fanboy","user_id":2}
{"create":{"_index":"us-tweet","_id":"14"}}
{"date":"2014-09-24","name":"John Smith","tweet":"How many more cheesy tweets do I have to write?","user_id":1}
# Don't Repeat Yourself
# 出错，从 7.0 开始，一个索引下一个类型
POST /website/_bulk
{ "index": {"_type": "blog" } }
{ "title": "User logged in" }

GET /website/
# 不要其他的元数据
GET /website/blog/123/_source

# 检索文档的一部分 
GET /website/blog/123?_source=title,text

GET /website/blog/123?pretty

POST /website/blog/
{
  "title": "My second Blog entry"  , 
  "text": "still tying this out .... ", 
  "date": "2014/01/02"
  
}

PUT /website/blog/123
{
  "title": "My First Blog entry"  , 
  "text": "just tying this out .... ", 
  "date": "2014/01/01"
  
}

PUT /website

GET _search
{
  "query": {
    "match_all": {}
  }
}

PUT /megacorp

PUT /megacorp/employee/1 
{     "first_name" : "John",     "last_name" :  "Smith",     "age" :        25,     "about" :      "I love to go rock climbing",     "interests": [ "sports", "music" ] 
  
}

PUT /megacorp/employee/2 
{     "first_name" :  "Jane",     "last_name" :   "Smith",     "age" :         32,     "about" :       "I like to collect rock albums",     "interests":  [ "music" ] }

PUT /megacorp/employee/3 
{     "first_name" :  "Douglas",     "last_name" :   "Fir",     "age" :         35,     "about":        "I like to build cabinets",     "interests":  [ "forestry" ] }

GET /megacorp/employee/1

GET /megacorp/employee/_search 

GET /megacorp/employee/_search?q=last_name:Smith 

# DSL 查询
GET /megacorp/employee/_search 
{
  "query": {
    "match": {
      "last_name": "Smitch"
    }
  }
}


# 年龄大于30岁的员工。

GET /megacorp/employee/_search 
{
  "query": {
    "bool":{
      "filter": {
        "range": {
          "age": { "gt": 30 }
        }
      },
      "must": {
        "match": {
          "last_name": "smitch"
        }
      }
    }
  }
}

GET /megacorp/employee/_search 
{
  "query": {
    "match": {
      "about": "rock climbing"
    }
  }
}


GET /megacorp/employee/_search 
{
  "query": {
    "match_phrase": {
      "about": "rock climbing"
    }
  }
}

# 因为我要聚合的字段 「interests」没有进行优化，也类似没有加索引 
# 没有优化的字段 es 默认是禁止聚合/排序操作的。 
PUT /megacorp/_mapping?pretty
{
  "properties": {
    "interests": {
      "type": "text", 
      "fielddata": true
    }
  }
}

# 让我们找到所有职员中最大的共同点是什么？
GET /megacorp/employee/_search
{
  "aggs":{
    "all_interests": {
      "terms": {"field": "interests" }
    }
  }
}


GET /megacorp/employee/_search 
{
  "query": {
    "match": {
      "last_name": "smith"
    }
  }, 
  "aggs": {
    "all_interests": {
      "terms": {
        "field": "interests"
      }
    }
  }
}

GET /megacorp/employee/_search
{
  "aggs": {
    "all_interests": {
      "terms": {
        "field": "interests"
      }, 
      "aggs": {
        "avg_age": {
          "avg": {"field": "age"} 
        }
      }
    }
  }
}


PUT /blogs
{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1
  }
}


GET /_cluster/health

import tornado.web
from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
import time

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write("Hello, world %s" % time.time())


class SleepHandler(tornado.web.RequestHandler):
    @property
    def executor(self):
        return self.application.executor

    @tornado.gen.coroutine
    def get(self, n):
        n = yield self._exe(n)
        self.write("Awake! %s" % time.time())
        self.finish()

    @run_on_executor
    def _exe(self, n):
        """
        This is a long time job and may block the server,such as a complex DB query or Http request.
        """
        time.sleep(float(n))
        return n


class App(tornado.web.Application):
    def __init__(self):
        handlers = [
                        (r"/", MainHandler),
                        (r"/sleep/(\d+)", SleepHandler),
                    ]
        tornado.web.Application.__init__(self, handlers)
        self.executor = ThreadPoolExecutor(max_workers=60)


if __name__ == "__main__":
    application = App()
    application.listen(8888)
    tornado.ioloop.IOLoop.instance().start()