import sys, os, time, json

def saveContext(filename,*name):

        format = '^'

        context = name[0]

        for i in name[1:]:

            context = context + format + str(i)

            context = str(context).replace('（','(').replace('）',')').replace('，',',').replace('：',':')

        #去除首位空格

        filename = filename.strip()

        #读取目录名称

        path = os.path.dirname(filename)

        #如果目录不存在则创建目录

        if not os.path.exists(path):

            os.makedirs(path)

        #读取文件名称

        name = os.path.basename(filename)

        fp = open(filename,'a')

        fp.write(context+'\n')

        fp.close()

#coding:utf-8

import sys, os, time, json

import urllib2

from pyquery import PyQuery as pq

from lxml import etree

from selenium import webdriver

from urlparse import urljoin

#设置utf-8模式

reload(sys)

sys.setdefaultencoding( "utf-8" )

#初始化创建浏览器

def init_drive():

    ua = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.3 Safari/537.36"

    cap = webdriver.DesiredCapabilities.PHANTOMJS

    cap["phantomjs.page.settings.resourceTimeout"] = 20000

    cap["phantomjs.page.settings.loadImages"] = True

    cap["phantomjs.page.settings.disk-cache"] = True

    cap["phantomjs.page.settings.userAgent"] = ua

    cap["phantomjs.page.customHeaders.User-Agent"] =ua

    cap["phantomjs.page.customHeaders.Referer"] = "http://tj.ac.10086.cn/login/"

    #driver = webdriver.PhantomJS(executable_path='/home/shutong/phantomjs/bin/phantomjs',desired_capabilities=cap, service_args=['--ignore-ssl-errors=true'])

    driver = webdriver.PhantomJS(desired_capabilities=cap, service_args=['--ignore-ssl-errors=true'])

    driver.set_page_load_timeout(60)

    driver.set_script_timeout(60)

    return driver

#初始化创建浏览器

driver = init_drive()

driver.get(url)

html = driver.page_source

#退出浏览器

driver.quit()

#coding:utf-8

import requests, json, time, re, os, sys, time

import urllib2

import random

import numpy as np

#设置为utf-8模式

reload(sys)

sys.setdefaultencoding( "utf-8" )

#最终获取url的数据

def getHtml(url):

    ua_list = ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv2.0.1) Gecko/20100101 Firefox/4.0.1","Mozilla/5.0 (Windows NT 6.1; rv2.0.1) Gecko/20100101 Firefox/4.0.1","Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11","Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"]

    user_agent = random.choice(ua_list)

    request = urllib2.Request(url)

    request.add_header("User-Agent",user_agent)

    response = urllib2.urlopen(request,data=None,timeout=60)

    html = response.read()

    #可以根据编码格式进行编码

    #html = unicode(html,'utf-8')

    return html

import time

import sys

import os

import shutil

import MySQLdb

import urllib2

from pyquery import PyQuery as pq

from lxml import etree

import urllib

import sys

import httplib

import datetime

import json

from selenium import webdriver

from urlparse import urljoin

httplib.HTTPConnection._http_vsn = 10

httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0'

#设置utf-8模式

reload(sys)

sys.setdefaultencoding( "utf-8" )

#获取常用时间格式的函数

#'%Y-%m-%d' 2017-11-18

#'%Y%m%d'   20171118

#%Y%m%d%H'  2017111817

#空或其他   2017-11-18 17:26:35

def getTime(*format):

    now = ''

    try:

        format = format[0]

    except :

        pass

    if format == '%Y-%m-%d':

        now = time.strftime('%Y-%m-%d',time.localtime(time.time()))

    elif format == '%Y%m%d':

        now = time.strftime('%Y%m%d',time.localtime(time.time()))

    elif format == '%Y%m%d%H':

        now = time.strftime('%Y%m%d%H',time.localtime(time.time()))

    else :

        now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))

    return now 

import time

import sys

import os

import shutil

import MySQLdb

import urllib2

from pyquery import PyQuery as pq

from lxml import etree

import urllib

import sys

import httplib

import datetime

import json

from selenium import webdriver

from urlparse import urljoin

httplib.HTTPConnection._http_vsn = 10

httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0'

#设置utf-8模式

reload(sys)

sys.setdefaultencoding( "utf-8" )

#定义MySql数据库连接

def conn_mysql(host='192.168.11.43',user='root',passwd='root',db='edw'):

    conn = ''

    try:

        conn= MySQLdb.connect(

        host= host,

        port = 3306,

        user=user,

        passwd=passwd,

        db =db,

        )

        #print "连接mysql成功"

    except :

        #pass

        print "连接mysql失败"

    return conn

#执行sql语句返回结果

def excute_sql(conn,sql):

    #conn = conn_mysql(host='192.168.122.194',user='root',passwd='123456',db='label')

    cur = conn.cursor()

    cur.execute('set character_set_client = utf8')

    cur.execute('set character_set_server = utf8')

    cur.execute('set character_set_connection = utf8')

    cur.execute('set character_set_results = utf8')

    cur.execute('set collation_connection = utf8_general_ci')

    cur.execute('set collation_server = utf8_general_ci')

    result = cur.fetchmany(cur.execute(sql))

    cur.close()

    conn.commit()

    conn.close()

    return result