字符相关
public class CharsetUtils {
private enum Charset {
/**
* 7位ASCII字符，也叫作ISO646-US、Unicode字符集的基本拉丁块
*/
US_ASCII("US-ASCII", "位ASCII字符，也叫作ISO646-US、Unicode字符集的基本拉丁块 "),
ISO_8859_1("ISO-8859-1", "ISO 拉丁字母表 No.1，也叫作 ISO-LATIN-1"),
GBK("GBK", "中文超大字符集"),
UTF_8("UTF-8", "8 位 UCS 转换格式"),
UTF_16BE("UTF-16BE", "16 位 UCS 转换格式，Big Endian（最低地址存放高位字节）字节顺序"),
UTF_16LE("UTF_16LE", "16 位 UCS 转换格式，Big Endian（最低地址存放高位字节）字节顺序"),
UTF_16("UTF_16", "16 位 UCS 转换格式，字节顺序由可选的字节顺序标记来标识");
private String encode;
private String desc;
public String getEncode() {
return encode;
}
public void setEncode(String encode) {
this.encode = encode;
}
public String getDesc() {
return desc;
}
public void setDesc(String desc) {
this.desc = desc;
}
private Charset(String encode, String desc) {
this.encode = encode;
this.desc = desc;
}
}
/**
* 获取传入字符串的编码格式
*
* @param str
* @return
*/
public static String getEncode(String str) throws UnsupportedEncodingException {
if (!StringUtils.isEmpty(str)) {
for (Charset charset : Charset.values()) {
String tryStr = new String(str.getBytes(charset.getEncode()), charset.getEncode());
if (str.equals(tryStr))
return charset.getEncode();
}
}
throw new UnsupportedEncodingException("编码库中不存在");
}
/**
* 字符串编码转换的实现方法
*
* @param str 待转换编码的字符串
* @param newCharset 目标编码
* @return
* @throws UnsupportedEncodingException
*/
public static String changeCharset(String str, String newCharset)
throws UnsupportedEncodingException {
if (str != null) {
//获取到原字符编码
String charsetName = getEncode(str);
//用默认字符编码解码字符串。
byte[] bs = str.getBytes(charsetName);
//用新的字符编码生成字符串
return new String(bs, newCharset);
}
return null;
}
public static String changeCharset(String str, String oldCharset, String newCharset)
throws UnsupportedEncodingException {
if (str != null) {
//获取到原字符编码
String charsetName = oldCharset;
//用默认字符编码解码字符串。
byte[] bs = str.getBytes(charsetName);
//用新的字符编码生成字符串
return new String(bs, newCharset);
}
return null;
}
/**
* 将字符编码转换成US-ASCII码
*/
public static String toASCII(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.US_ASCII.getEncode());
}
/**
* 将字符编码转换成ISO-8859-1码
*/
public static String toISO_8859_1(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.ISO_8859_1.getEncode());
}
/**
* 将字符编码转换成UTF-8码
*/
public static String toUTF_8(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.UTF_8.getEncode());
}
/**
* 将字符编码转换成UTF-16BE码
*/
public static String toUTF_16BE(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.UTF_16BE.getEncode());
}
/**
* 将字符编码转换成UTF-16LE码
*/
public static String toUTF_16LE(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.UTF_16LE.getEncode());
}
/**
* 将字符编码转换成UTF-16码
*/
public static String toUTF_16(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.UTF_16.getEncode());
}
/**
* 将字符编码转换成GBK码
*/
public static String toGBK(String str) throws UnsupportedEncodingException {
return changeCharset(str, Charset.GBK.getEncode());
}
}
坐标相关
object CoordinateTransformUtil {
val x_pi: Double = 3.14159265358979324 * 3000.0 / 180.0
// π
val pi = 3.1415926535897932384626
// 长半轴
val a = 6378245.0
// 扁率
val ee = 0.00669342162296594323
/**
* 百度坐标系(BD-09)转WGS坐标
*
* @param lng 百度坐标纬度
* @param lat 百度坐标经度
* @return WGS84坐标数组
*/
def bd09towgs84(lng: Double, lat: Double): Array[Double] = {
val gcj = bd09togcj02(lng, lat)
val wgs84 = gcj02towgs84(gcj(0), gcj(1))
wgs84
}
/**
* WGS坐标转百度坐标系(BD-09)
*
* @param lng WGS84坐标系的经度
* @param lat WGS84坐标系的纬度
* @return 百度坐标数组
*/
def wgs84tobd09(lng: Double, lat: Double): Array[Double] = {
val gcj = wgs84togcj02(lng, lat)
val bd09 = gcj02tobd09(gcj(0), gcj(1))
bd09
}
/**
* 火星坐标系(GCJ-02)转百度坐标系(BD-09)
* <p>
* 谷歌、高德——>百度
*
* @param lng 火星坐标经度
* @param lat 火星坐标纬度
* @return 百度坐标数组
*/
def gcj02tobd09(lng: Double, lat: Double): Array[Double] = {
val z = Math.sqrt(lng * lng + lat * lat) + 0.00002 * Math.sin(lat * x_pi)
val theta = Math.atan2(lat, lng) + 0.000003 * Math.cos(lng * x_pi)
val bd_lng = z * Math.cos(theta) + 0.0065
val bd_lat = z * Math.sin(theta) + 0.006
Array[Double](bd_lng, bd_lat)
}
/**
* 百度坐标系(BD-09)转火星坐标系(GCJ-02)
* <p>
* 百度——>谷歌、高德
*
* @param bd_lon 百度坐标纬度
* @param bd_lat 百度坐标经度
* @return 火星坐标数组
*/
def bd09togcj02(bd_lon: Double, bd_lat: Double): Array[Double] = {
val x = bd_lon - 0.0065
val y = bd_lat - 0.006
val z = Math.sqrt(x * x + y * y) - 0.00002 * Math.sin(y * x_pi)
val theta = Math.atan2(y, x) - 0.000003 * Math.cos(x * x_pi)
val gg_lng = z * Math.cos(theta)
val gg_lat = z * Math.sin(theta)
Array[Double](gg_lng, gg_lat)
}
/**
* WGS84转GCJ02(火星坐标系)
*
* @param lng WGS84坐标系的经度
* @param lat WGS84坐标系的纬度
* @return 火星坐标数组
*/
def wgs84togcj02(lng: Double, lat: Double): Array[Double] = {
if (out_of_china(lng, lat)) return Array[Double](lng, lat)
var dlat = transformlat(lng - 105.0, lat - 35.0)
var dlng = transformlng(lng - 105.0, lat - 35.0)
val radlat = lat / 180.0 * pi
var magic = Math.sin(radlat)
magic = 1 - ee * magic * magic
val sqrtmagic = Math.sqrt(magic)
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
dlng = (dlng * 180.0) / (a / sqrtmagic * Math.cos(radlat) * pi)
val mglat = lat + dlat
val mglng = lng + dlng
Array[Double](mglng, mglat)
}
/**
* GCJ02(火星坐标系)转GPS84
*
* @param lng 火星坐标系的经度
* @param lat 火星坐标系纬度
* @return WGS84坐标数组
*/
def gcj02towgs84(lng: Double, lat: Double): Array[Double] = {
if (out_of_china(lng, lat)) return Array[Double](lng, lat)
var dlat = transformlat(lng - 105.0, lat - 35.0)
var dlng = transformlng(lng - 105.0, lat - 35.0)
val radlat = lat / 180.0 * pi
var magic = Math.sin(radlat)
magic = 1 - ee * magic * magic
val sqrtmagic = Math.sqrt(magic)
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
dlng = (dlng * 180.0) / (a / sqrtmagic * Math.cos(radlat) * pi)
val mglat = lat + dlat
val mglng = lng + dlng
Array[Double](lng * 2 - mglng, lat * 2 - mglat)
}
/**
* 纬度转换
*
* @param lng
* @param lat
* @return
*/
def transformlat(lng: Double, lat: Double): Double = {
var ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + 0.1 * lng * lat + 0.2 * Math.sqrt(Math.abs(lng))
ret += (20.0 * Math.sin(6.0 * lng * pi) + 20.0 * Math.sin(2.0 * lng * pi)) * 2.0 / 3.0
ret += (20.0 * Math.sin(lat * pi) + 40.0 * Math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
ret += (160.0 * Math.sin(lat / 12.0 * pi) + 320 * Math.sin(lat * pi / 30.0)) * 2.0 / 3.0
ret
}
/**
* 经度转换
*
* @param lng
* @param lat
* @return
*/
def transformlng(lng: Double, lat: Double): Double = {
var ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + 0.1 * lng * lat + 0.1 * Math.sqrt(Math.abs(lng))
ret += (20.0 * Math.sin(6.0 * lng * pi) + 20.0 * Math.sin(2.0 * lng * pi)) * 2.0 / 3.0
ret += (20.0 * Math.sin(lng * pi) + 40.0 * Math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
ret += (150.0 * Math.sin(lng / 12.0 * pi) + 300.0 * Math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
ret
}
/**
* 判断是否在国内，不在国内不做偏移
*
* @param lng
* @param lat
* @return
*/
def out_of_china(lng: Double, lat: Double): Boolean = {
if (lng < 72.004 || lng > 137.8347) return true
else if (lat < 0.8293 || lat > 55.8271) return true
false
}
}
时间相关
object DateTimeUtils {
private val ft = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")
private[bigdata] val millis = (s: String) => {
DateTime.parse(s, ft).getMillis / 1000
}
private[bigdata] val inHour = (millis: Long, sh: Int, eh: Int) => {
val date = new DateTime(millis)
val hour = date.hourOfDay.get
hour >= sh && hour < eh
}
//每天的毫秒数
private val day = 1000 * 60 * 60 * 24
//日期字符串的格式
private val format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
/**
* 返回前一天的 00:00:00 字符串格式
*
* @return
*/
def getLastDayStart: String = format.format(getTodayStartMills - day)
/**
* 返回前一天的 23:59:59 字符串格式
*
* @return
*/
def getLastDayEnd: String = format.format(getTodayStartMills - 1)
/**
* 返回当日的 00:00:00 字符串格式
*
* @return
*/
def getTodayStart: String = {
val zero = getTodayStartMills
format.format(zero)
}
/**
* 返回当日的 23:59:59 字符串格式
*
* @return
*/
def getTodayEnd: String = format.format(getTodayStartMills + day - 1)
/**
* 返回当日的 00:00:00 毫秒格式
*
* @return
*/
def getTodayStartMills: Long = {
val current = System.currentTimeMillis
val zero = ((current + TimeZone.getDefault.getRawOffset) / day * day) - TimeZone.getDefault.getRawOffset
zero
}
/**
* 返回前一天的 00:00:00 毫秒格式
*
* @return
*/
def getLastDayStartMills: Long = getTodayStartMills - day
def main(args: Array[String]): Unit = {
val inputPath = "/data/origin/vehicle/gps/taxi/440300/jiaowei/2019/10/08"
// println(path.substring(path.lastIndexOf("/")-7))
// val inputPath="/data/origin/vehicle/gps/taxi/440300/jiaowei/2019/10/09"
// val date = DateTime.parse(inputPath.substring(inputPath.lastIndexOf("/") - 7), DateTimeFormat.forPattern("yyyy/MM/dd")).plusDays(1)
// println(date.toString("yyyyMMdd HH:mm:ss"))
val today = DateTime.parse(inputPath.substring(inputPath.lastIndexOf("/") - 7), DateTimeFormat.forPattern("yyyy/MM/dd")).plusDays(1)
val time_tmp = "2019-10-07 23:59:22"
//loc_time 定位日期 整型 10 否 unix10位时间戳
val dataTime = DateTime.parse(time_tmp, DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"))
val isOk = dataTime.getMillis >= today.minusDays(1).getMillis && dataTime.getMillis < today.getMillis
println(isOk)
}
}
加密
object DesUtils {
def bytes2hex(bytes: Array[Byte]): String = {
val hex = new StringBuilder()
for (i <- 0 to bytes.length - 1) {
val b = bytes(i)
var negative = false
if (b < 0) {
negative = true
}
val inte = Math.abs(b)
val temp = Integer.toHexString(inte & 0xFF)
if (temp.length() == 1) {
hex.append("0")
}
// hex.append(temp.toLowerCase())
hex.append(temp)
}
hex.toString
}
// 生成AES密鈅
@throws(classOf[Exception])
def genKeyAES(): String = {
val pkey = "123"
val kgen = KeyGenerator.getInstance("DES")
kgen.init(56, new SecureRandom(pkey.getBytes))
val secretKey = kgen.generateKey
val enCodeFormat = secretKey.getEncoded
val key = new SecretKeySpec(enCodeFormat, "DES")
// val cipher = Cipher.getInstance("DES") // 创建密码器
// val keyGen = KeyGenerator.getInstance("DES")
// keyGen.init(56)
// val key = keyGen.generateKey()
val base64Str = Base64.encodeBase64String(key.getEncoded())
base64Str
}
@throws(classOf[Exception])
def loadKeyAES(base64Key: String): SecretKey = {
val bytes = Base64.decodeBase64(base64Key)
val key = new SecretKeySpec(bytes, "DES")
return key
}
def encryt(content: String): Array[Byte] = {
encrytAES(content.getBytes(), loadKeyAES(genKeyAES))
// bytes2hex(encrytAES(content.getBytes(), loadKeyAES(genKeyAES)))
}
def encryt2Str(content: String): String = {
bytes2hex(encrytAES(content.getBytes(), loadKeyAES(genKeyAES)))
}
@throws(classOf[Exception])
def encrytAES(source: Array[Byte], key: SecretKey): Array[Byte] = {
val cipher = Cipher.getInstance("DES")
cipher.init(Cipher.ENCRYPT_MODE, key)
cipher.doFinal(source)
}
def decry(source: Array[Byte]): String = {
new String(decryptAES(source, loadKeyAES(genKeyAES)))
}
// def decry2(source: String): String = {
// new String(decryptAES(hexStr2Byte(source), loadKeyAES(genKeyAES)))
// }
@throws(classOf[Exception])
def decryptAES(source: Array[Byte], key: SecretKey): Array[Byte] = {
val cipher = Cipher.getInstance("DES")
cipher.init(Cipher.DECRYPT_MODE, key)
cipher.doFinal(source)
}
def main(args: Array[String]): Unit = {
val content = "张三"
println(decry(encryt(content)))
}
}
object MD5Utils {
def encode32(text: String): String = {
val md = MessageDigest.getInstance("MD5")
md.update(text.getBytes)
val b = md.digest
var i = 0
val buf = new StringBuffer("")
var offset = 0
while ( {
offset < b.length
}) {
i = b(offset)
if (i < 0) i += 256
if (i < 16) buf.append("0")
buf.append(Integer.toHexString(i))
{
offset += 1;
offset - 1
}
}
buf.toString
}
def encode16(text: String): String = {
encode32(text).substring(8, 24)
}
/** *
* 任意文件转换成Md5
* Can convert a text to MD5
*
* @param inputStream
* @return md5
*/
def encode(inputStream: InputStream): String = {
var in = inputStream
try {
val digester = MessageDigest.getInstance("MD5")
val bytes = new Array[Byte](8192)
var byteCount = 0
while ( {
(byteCount = in.read(bytes));
byteCount > 0
}) digester.update(bytes, 0, byteCount)
val digest = digester.digest
val sb = new StringBuffer
for (b <- digest) {
val a = b & 0xff
var hex = Integer.toHexString(a)
if (hex.length == 1) hex = 0 + hex
sb.append(hex)
}
return sb.toString
} catch {
case e: Exception =>
e.printStackTrace()
} finally if (in != null) {
try
in.close
catch {
case e: IOException =>
e.printStackTrace()
}
in = null
}
null
}
}
GIS计算
object Spatial {
private val EARTH_RADIUS = 6378.137 // 地球半径
// 使用经纬度估算两点之间的距离(米)
private[bigdata] val distance = (lon1: Float, lat1: Float, lon2: Float, lat2: Float) => {
val (l1, l2) = (rad(lat1), rad(lat2))
val a = l1 - l2
val b = rad(lon1) - rad(lon2)
val d = EARTH_RADIUS * 2 * math.asin(math.sqrt(math.pow(math.sin(a / 2), 2) +
math.cos(l1) * math.cos(l2) * math.pow(math.sin(b / 2), 2)))
math.round(d * 10000) / 10
}
// 弧度到角度(PI=180°)
private def rad(d: Float): Float = d * math.Pi.toFloat / 180
}
其他
object StdUtils {
def isCnLngLat(lng: Double, lat: Double): Boolean = {
//纬度3.86~53.55，经度73.66~135.05
lng >= 73.66 && lng <= 135.05 && lat >= 3.86 && lat <= 53.55
}
def fillTime(date: String, pattern: String = "yyyy-MM-dd HH:mm:ss"): String = {
if (StdUtils.isTime(date, Array(pattern))) {
return date
}
var time1 = date
if (date.length > 19) {
time1 = date.substring(0, 19)
return time1
}
val time2 = "1971-01-01 00:00:00"
time1 + time2.substring(time1.length)
}
def main(args: Array[String]): Unit = {
println(fillTime("2018-10-01 00:02:35.000"))
}
def isMonth(str: String): Boolean = {
val parsePatterns = Array("yyyy/MM", "yyyy-MM", "yyyyMM")
return isTime(str, parsePatterns)
}
// val parsePatterns = Array("yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy/MM/dd", "yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyyMMdd")
def isDate(str: String): Boolean = {
val parsePatterns = Array("yyyy/MM/dd", "yyyy-MM-dd", "yyyyMMdd")
return isTime(str, parsePatterns)
}
def isHour(str: String): Boolean = {
val parsePatterns = Array("yyyy/MM/dd/HH", "yyyy-MM-dd HH", "yyyyMMdd HH")
return isTime(str, parsePatterns)
}
// def main(args: Array[String]): Unit = {
// val res = isTime("20571501040057", Array("yyyyMMddHHmmss"))
// println(res)
// }
def isTime(str: String, parsePatterns: Array[String]): Boolean = {
try {
val date = DateUtils.parseDate(str, null, parsePatterns: _*)
if (date.getTime > DateTime.now().getMillis) {
return false;
}
} catch {
case e: Exception => return false
}
return true;
}
def replaceBlank(str: String) = {
str.replaceAll(" ", "")
}
/**
* 临时替代数据年份
*
* @param date
* @param year
* @return
*/
def replaceYear(date: String, year: String = "2017"): String = {
if (!date.startsWith(year)) {
year + date.substring(4)
} else {
date
}
}
/**
* 同济需求
*
* @param date
* @param yearMonth
* @return
*/
def replaceYearMonth(date: String, yearMonth: String = "201710"): String = {
if (date.startsWith("20190312")) {
return "20171023"
}
if (!date.startsWith(yearMonth)) {
yearMonth + date.substring(6)
} else {
date
}
}
// def main(args: Array[String]): Unit = {
// val arr = "2939131247,19980077,-15093343,201,2017-01-01 15:34:23,1,0,0,1,0,0,0,0,0,0,0,0,131856,131856,131856,131856,16.987055,16.987055,16.987055,16.987055,866,866,866,866,130896,130896,130896,130896,4.746764,4.746764,4.746764,4.746764,0,0,0,0,0,0,1".split(",")
//
// println(arr)
// println(arr.length)
// }
/**
* 字符串前补0
*
* @param fillsize
* @param data
* @return
*/
def fillZero(fillsize: Int, data: String): String = {
val prefix = new StringBuilder
Range(0, fillsize - data.trim.length).foreach(x => {
prefix.append("0")
})
prefix.append(data.trim).toString()
}
/**
* 判断全不为空
*
* @param arr
* @return
*/
def allNotEmpty(arr: Array[String]): Boolean = {
var res = true
for (e <- arr if res) {
res = StringUtils.isNotEmpty(e)
}
res
}
}