# 确定哪些包是安装好的

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import datetime
from tqdm import tqdm
from sklearn.preprocessing import LabelEncoder
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.preprocessing import MinMaxScaler
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostRegressor
import warnings
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, log_loss
warnings.filterwarnings('ignore')

data_train =pd.read_csv('./train.csv')
data_test_a = pd.read_csv('./testA.csv')

data_train.info()
is_default = data_train['isDefault']

# 找到数值型的特征
numerical_fea  = list(data_train.select_dtypes(exclude=['object']).columns)
# 找到非结构化的特征
category_fea = list(filter(lambda x: x not in numerical_fea, list(data_train.columns)))
# 移除‘isDefault’特征
numerical_fea.remove('isDefault')

print(len(numerical_fea))

# 显示缺失值的情况
data_train.isnull().sum()

fill_type = 0
if fill_type == 1:
    #  缺失值替换为指定的值
    data_train = data_train.fillna(0)
elif fill_type == 2:
    # 用上面的值替换缺失值
    data_train = data_train.fillna(axis=0,method='ffill')
elif fill_type == 3:
    # 纵向用下面的值来替换，且最多只填充两个连续的
    data_train = data_train.fillna(axis=0,method='bfill',limit=2)

#按照平均数填充数值型特征
data_train[numerical_fea] = data_train[numerical_fea].fillna(data_train[numerical_fea].median())
data_test_a[numerical_fea] = data_test_a[numerical_fea].fillna(data_train[numerical_fea].median())
#按照众数填充类别型特征
data_train[category_fea] = data_train[category_fea].fillna(data_train[category_fea].mode())
data_test_a[category_fea] = data_test_a[category_fea].fillna(data_train[category_fea].mode())

# 处理没有结构化的数据 'issueDate' 
data_train.isnull().sum()
# 由于有些数据类型比如时间需要特别处理：['issueDate'] ['earliesCreditLine']
for data in [data_train, data_test_a]:
    data['issueDate'] = pd.to_datetime(data['issueDate'],format='%Y-%m-%d')
    startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
    #构造时间特征
    data['issueDateDT'] = data['issueDate'].apply(lambda x: x-startdate).dt.days

data_train['employmentLength'].value_counts(dropna=False).sort_index()
# employmentLength 这一列特征中主要存储的是带信息的字符串稍微处理下变成int类型
def employmentLength_to_int(s):
    if pd.isnull(s):
        return s
    else:
        return np.int8(s.split()[0])
for data in [data_train, data_test_a]:
    data['employmentLength'].replace(to_replace='10+ years', value='10 years', inplace=True)
    data['employmentLength'].replace('< 1 year', '0 years', inplace=True)
    data['employmentLength'] = data['employmentLength'].apply(employmentLength_to_int)

data['employmentLength'].value_counts(dropna=False).sort_index()
# 可以看出数据的分布主要还是集中在10年以上

data_train['earliesCreditLine'].sample(5)

# 对于这个类型的特征由于时间跨度大所以只需要年份就行了
for data in [data_train, data_test_a]:
    data['earliesCreditLine'] = data['earliesCreditLine'].apply(lambda s: int(s[-4:]))

semantic_dict = {
id:'为贷款清单分配的唯一信用证标识',
'loanAmnt':'贷款金额',
'term' :'贷款期限（year）',
'interestRate':'贷款利率',
'installment' :'分期付款金额',
'grade' :'贷款等级',
'subGrade' :'贷款等级之子级',
'employmentTitle' :'就业职称',
'employmentLength' :'就业年限（年）',
'homeOwnership' :'借款人在登记时提供的房屋所有权状况',
'annualIncome' :'年收入',
'verificationStatus' :'验证状态',
'issueDate' :'贷款发放的月份',
'purpose' :'借款人在贷款申请时的贷款用途类别',
'postCode' :'借款人在贷款申请中提供的邮政编码的前3位数字',
'regionCode' :'地区编码',
'dti' :'债务收入比',
'delinquency_2years' :'借款人过去2年信用档案中逾期30天以上的违约事件数',
'ficoRangeLow':'借款人在贷款发放时的fico所属的下限范围',
'ficoRangeHigh':'借款人在贷款发放时的fico所属的上限范围',
'openAcc':'借款人信用档案中未结信用额度的数量',
'pubRec':'贬损公共记录的数量',
'pubRecBankruptcies':'公开记录清除的数量',
'revolBal':'信贷周转余额合计',
'revolUtil':'循环额度利用率，或借款人使用的相对于所有可用循环信贷的信贷金额',
'totalAcc':'借款人信用档案中当前的信用额度总数',
'initialListStatus':'贷款的初始列表状态',
'applicationType':'表明贷款是个人申请还是与两个共同借款人的联合申请',
'earliesCreditLine':'借款人最早报告的信用额度开立的月份',
'title':'借款人提供的贷款名称',
'policyCode':'公开可用的策略_代码=1新产品不公开可用的策略_代码=2n系列匿名特征 匿名特征n0-n14，为一些贷款人行为计数特征的处理',}

# 把grad映射一下
for data in [data_train, data_test_a]:
    data['grade'] = data['grade'].map({'A':1,'B':2,'C':3,'D':4,'E':5,'F':6,'G':7})
for data in [data_train, data_test_a]:
    data = pd.get_dummies(data, columns=['subGrade', 'homeOwnership', 'verificationStatus', 'purpose', 'regionCode'], drop_first=True)

# 使用3sigma分析来找异常值，此时的data是[200000, 148], 增加的列编程了
def find_outliers_by_3segama(data,fea):
    data_std = np.std(data[fea])
    data_mean = np.mean(data[fea])
    outliers_cut_off = data_std * 3
    lower_rule = data_mean - outliers_cut_off
    upper_rule = data_mean + outliers_cut_off
    data[fea+'_outliers'] = data[fea].apply(lambda x:str('异常值') if x > upper_rule or x < lower_rule else '正常值')
    return data

data_train = data_train.copy()
for fea in numerical_fea:
    data_train = find_outliers_by_3segama(data_train,fea)
    print(data_train[fea+'_outliers'].value_counts())
    print(data_train.groupby(fea+'_outliers')['isDefault'].sum())
    print('*'*10)
# data_train[80w, 89]

#删除异常值对应的行
for fea in numerical_fea:
    data_train = data_train[data_train[fea+'_outliers']=='正常值']
    data_train = data_train.reset_index(drop=True)

# 通过除法映射到间隔均匀的分箱中，每个分箱的取值范围都是loanAmnt/1000
data['loanAmnt_bin1'] = np.floor_divide(data['loanAmnt'], 1000)
## 通过对数函数映射到指数宽度分箱
data['loanAmnt_bin2'] = np.floor(np.log10(data['loanAmnt']))
## 分位数分箱
data['loanAmnt_bin3'] = pd.qcut(data['loanAmnt'], 10, labels=False)

#label-encode:subGrade,postCode,title
# 高维类别特征需要进行转换
for col in tqdm(['employmentTitle', 'postCode', 'title','subGrade']):
    le = LabelEncoder()
    le.fit(list(data_train[col].astype(str).values) + list(data_test_a[col].astype(str).values))
    data_train[col] = le.transform(list(data_train[col].astype(str).values))
    data_test_a[col] = le.transform(list(data_test_a[col].astype(str).values))
print('Label Encoding 完成')

"纵向用缺失值上面的值替换缺失值"
data_train = data_train.fillna(axis=0,method='ffill')

x_train = data_train
#计算协方差
data_corr = x_train.corrwith(data_train.isDefault) #计算相关性
result = pd.DataFrame(columns=['features', 'corr'])
result['features'] = data_corr.index
result['corr'] = data_corr.values

# 当然也可以直接看图
data_numeric = data_train[numerical_fea]
correlation = data_numeric.corr()

f , ax = plt.subplots(figsize = (7, 7))
plt.title('Correlation of Numeric Features with Price',y=1,size=16)
sns.heatmap(correlation,square = True,  vmax=0.8)

features = [f for f in data_train.columns if f not in ['id','issueDate','isDefault'] and '_outliers' not in f]
x_train = data_train[features]
x_test = data_test_a[features]
y_train = data_train['isDefault']

def cv_model(clf, train_x, train_y, test_x, clf_name):
    folds = 5
    seed = 2020
    kf = KFold(n_splits=folds, shuffle=True, random_state=seed)

    train = np.zeros(train_x.shape[0])
    test = np.zeros(test_x.shape[0])

    cv_scores = []

    for i, (train_index, valid_index) in enumerate(kf.split(train_x, train_y)):
        print('************************************ {} ************************************'.format(str(i+1)))
        trn_x, trn_y, val_x, val_y = train_x.iloc[train_index], train_y[train_index], train_x.iloc[valid_index], train_y[valid_index]

        if clf_name == "lgb":
            train_matrix = clf.Dataset(trn_x, label=trn_y)
            valid_matrix = clf.Dataset(val_x, label=val_y)

            params = {
                'boosting_type': 'gbdt',
                'objective': 'binary',
                'metric': 'auc',
                'min_child_weight': 5,
                'num_leaves': 2 ** 5,
                'lambda_l2': 10,
                'feature_fraction': 0.8,
                'bagging_fraction': 0.8,
                'bagging_freq': 4,
                'learning_rate': 0.1,
                'seed': 2020,
                'nthread': 28,
                'n_jobs':24,
                'silent': True,
                'verbose': -1,
            }

            model = clf.train(params, train_matrix, 50000, valid_sets=[train_matrix, valid_matrix], verbose_eval=200,early_stopping_rounds=200)
            val_pred = model.predict(val_x, num_iteration=model.best_iteration)
            test_pred = model.predict(test_x, num_iteration=model.best_iteration)
            
            # print(list(sorted(zip(features, model.feature_importance("gain")), key=lambda x: x[1], reverse=True))[:20])
                
        if clf_name == "xgb":
            train_matrix = clf.DMatrix(trn_x , label=trn_y)
            valid_matrix = clf.DMatrix(val_x , label=val_y)
            
            params = {'booster': 'gbtree',
                      'objective': 'binary:logistic',
                      'eval_metric': 'auc',
                      'gamma': 1,
                      'min_child_weight': 1.5,
                      'max_depth': 5,
                      'lambda': 10,
                      'subsample': 0.7,
                      'colsample_bytree': 0.7,
                      'colsample_bylevel': 0.7,
                      'eta': 0.04,
                      'tree_method': 'exact',
                      'seed': 2020,
                      'nthread': 36,
                      "silent": True,
                      }
            
            watchlist = [(train_matrix, 'train'),(valid_matrix, 'eval')]
            
            model = clf.train(params, train_matrix, num_boost_round=50000, evals=watchlist, verbose_eval=200, early_stopping_rounds=200)
            val_pred  = model.predict(valid_matrix, ntree_limit=model.best_ntree_limit)
            test_pred = model.predict(test_x , ntree_limit=model.best_ntree_limit)
                 
        if clf_name == "cat":
            params = {'learning_rate': 0.05, 'depth': 5, 'l2_leaf_reg': 10, 'bootstrap_type': 'Bernoulli',
                      'od_type': 'Iter', 'od_wait': 50, 'random_seed': 11, 'allow_writing_files': False}
            
            model = clf(iterations=20000, **params)
            model.fit(trn_x, trn_y, eval_set=(val_x, val_y),
                      cat_features=[], use_best_model=True, verbose=500)
            
            val_pred  = model.predict(val_x)
            test_pred = model.predict(test_x)
            
        train[valid_index] = val_pred
        test = test_pred / kf.n_splits
        cv_scores.append(roc_auc_score(val_y, val_pred))
        
        print(cv_scores)
        
    print("%s_scotrainre_list:" % clf_name, cv_scores)
    print("%s_score_mean:" % clf_name, np.mean(cv_scores))
    print("%s_score_std:" % clf_name, np.std(cv_scores))
    return train, test

def lgb_model(x_train, y_train, x_test):
    lgb_train, lgb_test = cv_model(lgb, x_train, y_train, x_test, "lgb")
    return lgb_train, lgb_test

def xgb_model(x_train, y_train, x_test):
    xgb_train, xgb_test = cv_model(xgb, x_train, y_train, x_test, "xgb")
    return xgb_train, xgb_test

def cat_model(x_train, y_train, x_test):
    cat_train, cat_test = cv_model(CatBoostRegressor, x_train, y_train, x_test, "cat")

lgb_train, lgb_test = lgb_model(x_train, y_train, x_test)

testA_result = pd.read_csv('./testA.csv')
roc_auc_score(testA_result['isDefault'].values, lgb_test)

def twoSum(self, nums: List[int], target: int) -> List[int]:
    for i in range(len(nums)):
        for j in range(i+1 , len(nums), 1):
            if nums[j] == target - nums[i]:
                return [i , j]

def twoSum(self, nums: List[int], target: int) -> List[int]:
	hash_tab = {}
       for i in range(len(nums)):
           if target - nums[i] in hash_tab.keys():
               return [i, hash_tab[target - nums[i]]]
           else:
               hash_tab[nums[i]] = i

def threeSum(self, nums: List[int]) -> List[List[int]]:
    res = []
    nums.sort()
    target = 0
    for i in range(len(nums)):
        if nums[i] > 0: break
        if i != 0 and nums[i] == nums[i-1]:
            continue
        pre = i + 1
        end = len(nums) - 1

        while pre < end:
            sum = nums[end] + nums[pre]
            if -nums[i] > sum:
                pre += 1
            elif -nums[i] < sum:
                end -= 1
            elif -nums[i] == sum:
                res.append([nums[i], nums[pre], nums[end]])
                pre += 1
                end -= 1
                while pre < end and nums[pre] == nums[pre - 1]:
                    pre += 1
                while pre < end and nums[end] == nums[end + 1]:
                    end -= 1
    return res

def threeSumClosest(self, nums: List[int], target: int) -> int:
    nums.sort()
    temp = nums[0] + nums[1] + nums[2]
    res = abs(nums[0] + nums[1] + nums[2] - target)
    for i in range(len(nums) - 2):
        if i != 0 and nums[i] == nums[i - 1]:
            continue
        pre = i + 1
        end = len(nums) - 1
        while pre < end:
            t_sum = nums[pre] + nums[i] + nums[end]
            if t_sum == target:
                return target
            else:
                if abs(target - t_sum) < res:
                    res = abs(target - t_sum)
                    temp = t_sum
            if t_sum < target:
                pre += 1
            else:
                end -= 1

    return temp

def fourSum(self, nums: List[int], target: int) -> List[List[int]]:
    res = []
    nums.sort()
    for i in range(len(nums) - 3):
        if i != 0 and nums[i] == nums[i-1]:
            continue
        for j in range(i+1, len(nums)-2, 1):

            if j > i+1 and nums[j] == nums[j-1]:
                continue
            pre = j + 1
            end = len(nums) - 1

            while pre < end:
                sum = nums[end] + nums[pre] +  nums[i] + nums[j]
                if target > sum:
                    pre += 1
                elif target < sum:
                    end -= 1
                elif target == sum:
                    res.append([nums[i], nums[j], nums[pre], nums[end]])
                    pre += 1
                    end -= 1
                    while pre < end and nums[pre] == nums[pre - 1]:
                        pre += 1
                    while pre < end and nums[end] == nums[end + 1]:
                        end -= 1
    return res

def intersection(self, nums1: List[int], nums2: List[int])
    set1 = set(nums1)
    set2 = set(nums2)
    return [i for i in set1 if i in set2]

def intersection(self, nums1: List[int], nums2: List[int]) 
    return set(nums1)&set(nums2)

def isHappy(self, n: int) -> bool:
    num_table = set()
    while n >= 1 and n not in num_table:
        sum = 0
        num_table.add(n)
        while n > 0:
            n, mod = divmod(n, 10)
            sum += mod ** 2
        if sum == 1:
            return True
        else:
            n = sum
    return False

def wordPattern(self, pattern: str, str: str) -> bool:
   str_list = str.split(' ')
   if len(pattern) != len(str_list):
       return False
   hash_table = {}
   for i in range(len(pattern)):
       if pattern[i] not in hash_table.keys():
           if str_list[i] in hash_table.values():
               return False
           else:
               hash_table[pattern[i]] = str_list[i]
       elif hash_table[pattern[i]] != str_list[i]:
           return False
   return True

def frequencySort(self, s: str) -> str:
    from collections import Counter
    s1 = Counter(s)

    s2 = sorted(s1.items(),key=lambda item:item[1],reverse=True)

    res = ''
    for key,value in s2:
        res += key*value
    return res

def isAnagram(self, s: str, t: str) -> bool:
    if s is None or t is None:
        return False
    if len(s) != len(t):
        return False

    from collections import Counter
    cnt1 = Counter(s)
    cnt2 = Counter(t)
    char_tab = cnt2.keys()
    for char in cnt1.keys():
        if char not in char_tab:
            return False
        elif cnt2[char] != cnt1[char]:
            return False
    return True

def isIsomorphic(self, s: str, t: str) -> bool:
    return list(map(s.index,s)) == list(map(t.index,t))

def lengthOfLIS(self, nums) -> int:
    max_len = 0
    if not nums:
            return max_len
    for i in range(len(nums)):
        tmp_max_num = nums[i]
        tmp_max_num2 = nums[i]
        res = []
        res.append(tmp_max_num)
        for j in range(i+1, len(nums), 1):
            if nums[j] > tmp_max_num:
                tmp_max_num2 = tmp_max_num
                tmp_max_num = nums[j]
                res.append(nums[j])
                elif nums[j] > tmp_max_num2:
                    res.remove(tmp_max_num)
                    res.append(nums[j])
                    tmp_max_num = nums[j]
		max_len = max(max_len, len(res))
	return max_len

def lengthOfLIS(self, nums) -> int:
	if not nums or not len(nums):
           return 0
       records = []
       records.append(nums[0])
       for i in range(1, len(nums)):
           if nums[i]>records[-1]:
               records.append(nums[i])
           else:
               j = len(records)-1
               while j>=0:
                   if records[j]>=nums[i]:
                       j-=1
                   else:
                       break
               records[j+1]=nums[i]
       return len(records)

class Solution:
    def findLengthOfLCIS(self, nums: List[int]) -> int:
        max_len = 0
        if nums is None : return max_len
        if len(nums) > 1 :
            max_len += 1
        elif len(nums) == 0:
            return 0
        else:
            return 1
        max_all = 0
        for i in range(1, len(nums)):
            if nums[i] > nums[i-1]:
                max_len += 1
            else:
                max_len = 1
            max_all = max(max_all, max_len)
        return max_all

class Solution:
    def longestPalindrome(self, s: str) -> str:
        len_s = len(s)
        res = ''
        max_len = 0
        dp = [[False] * len_s for _ in range(len_s)]
        for i in range(len_s-1, -1, -1):
            for n in range(len_s - i):
                j = i+n
                if n == 0:
                    dp[i][j] = True
                elif n == 1:
                    dp[i][j] = True if s[i] == s[j] else False
                else:
                    dp[i][j] = (dp[i+1][j-1] and s[i] == s[j])
                if dp[i][j] and n+1 > max_len:
                    max_len = n+1
                    res = s[i:j+1]
        return res

class Solution:
    def longestPalindromeSubseq(self, s: str) -> int:
        len_s = len(s)
        dp = [[0] * len_s for _ in range(len_s)]
        for i in range(len_s):
            dp[i][i] = 1
        for i in range(len_s, -1, -1):
            for j in range(i + 1, len_s):
                # print(s[i:j+1])
                if s[i] == s[j]:
                    dp[i][j] = dp[i + 1][j - 1] + 2
                else:
                    dp[i][j] = max(dp[i][j - 1], dp[i + 1][j])
        return dp[0][-1]

def minDistance(self, word1, word2):
        m=len(word1) 
        n=len(word2)
        dp=[[0 for _ in range(n+1)] for _ in range(m+1)]
        #空字符串的初始化
        for i in range(n+1):
            dp[0][i]=i 
        for j in range(m+1):
            dp[j][0]=j 
        # 每一次一个指针走一步    
        for i in range(1,m+1): 
            for j in range(1,n+1):
                if word1[i-1]==word2[j-1]: 
                    dp[i][j]=dp[i-1][j-1]
                else:
                    #不同情况递归求下一个状态，分治思想
                    dp[i][j]=min(dp[i-1][j],dp[i][j-1],dp[i-1][j-1])+1 
        return dp[-1][-1]

class Solution:
    def myPow(self, x: float, n: int) -> float:
        # 数据预处理
        res = 1.0 
        if n ==0:
            return res
        if n < 0:
            x = 1 /x
            n = -n 
        # 问题划分
        if n%2 == 0:
            res = self.myPow(x*x, n/2)
            return res
        else:
            res1 = self.myPow(x, n-1)
            # 集合所有的结果
            res = res1*x
            return res

def myPow(self, x: float, n: int) -> float:
    # 数据预处理
    res = 1.0 
    if n ==0:
        return res
    if n < 0:
        x = 1 /x
        n = -n 
    # 问题划分
    res1 = self.myPow(x, n-1)
    # 集合所有的结果
    res = res1*x
    return res

def myPow(self, x: float, n: int) -> float:
    # 数据预处理
    res = 1.0 
    if n ==0:
        return res
    if n == 1:
        return x
    if n == -1:
        return 1.0/x
    if n < 0:
        x = 1 /x
        n = -n 
    # 问题划分
    if n%2 == 0:
        res = self.myPow(x*x, n//2)
        return res
    else:
        res1 = self.myPow(x*x, (n-1)//2)
        # 集合所有的结果
        res = res1*x
        return res

class Solution:
    def maxSubArray(self, nums: List[int]) -> int:
        # 结束条件
        if len(nums) == 1:
            return nums[0]
        if nums is None or len(nums) == 0:
            return None
        # 划分子问题
        max_sum_l = self.maxSubArray(nums[:len(nums)//2])
        max_sum_r = self.maxSubArray(nums[len(nums)//2:])
        # 从中间开始
        m_left = nums[len(nums)//2 - 1]
        m_right = nums[len(nums)//2]
        temp = 0
        for i in range(len(nums)//2 - 1, -1, -1):
            temp += nums[i]
            m_left = max(m_left, temp)
        temp = 0
        for i in range(len(nums)//2, len(nums)):
            temp += nums[i]
            m_right = max(m_right, temp)
        return max(max_sum_l, max_sum_r, m_right+m_left)

class Solution:
    def maxSubArray(self, nums: List[int]) -> int:
        dp = [0] * len(nums)
        # 存储每一步的最优结果
        dp[0] = nums[0]
        max_num = nums[0]
        for i in range(1, len(nums)):
            # 对于每一次更新加入i个数字后的序列的最佳子序列和
            dp[i] = max(dp[i-1]+nums[i], nums[i])
            if dp[i] > max_num:
                max_num = dp[i]
        return max_num

class Solution:
    def majorityElement(self, nums: List[int]) -> int:
        table = {}
        for i in nums:
            if i in table.keys():
                table[i] += 1
            else:
                table[i] = 1
        max_freq = 0

        for key in table.keys():
            if table[key] > max_freq:
                max_freq = table[key]
                temp = key
        return temp

def majorityElement(self, nums) -> int:
    if nums is None:
        return None
    if len(nums) == 1:
        return nums[0]
    left = self.majorityElement(nums[: len(nums) // 2])
    right = self.majorityElement(nums[len(nums) // 2:])
    if right == left:
        return right
    else:
        # N+N
        return right if nums.count(right) > nums.count(left) else left

from collections import Counter
   def majorityElement(self, nums) -> int:
        c = Counter(nums)
        return c.most_common(1)[0][0]

import streamlit as st
import pydeck as pdk
import pandas as pd
from urllib import request
import json
import re

def get_data_from_web():
    user_agent = "ritarma/1.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0"
    headers = {"User-Agent": user_agent}
    req = request.Request(
        "https://3g.dxy.cn/newh5/view/pneumonia?scene=2&clicktime=1579596108&enterid=1579596108&from=groupmessage"
        "&isappinstalled=0",
        headers=headers)
    resp = request.urlopen(req)
    # 打开页面
    try:
        html = resp.read()
    except http.client.IncompleteRead as e:
        html = e.partial

    want = re.compile(r' window.getAreaStat.*?</script>', re.S)
    result = want.findall(str(html.decode("utf-8")))
    result = str(result).replace(" ", "")
    # 处理结果
    result_p = result
    # print('处理结果:',result)
    result_p = result_p.replace(']}catch(e){}</script>', " ")
    result_p = result_p.replace('}', "}\n")
    result_p = result_p.replace('window.getAreaStat=[', " ")
    result_p = result_p.replace("[' ", "")
    result_p = result_p.replace(" ']", "")
    json_output = json.loads('[' + result_p + ']')
    return json_output
`

def get_lat_by_name(name: str, pos_dic):
    pos_data = str(pos_dic[name]).split(',')
    lat = float(pos_data[0])
    lon = float(pos_data[1])
    return lat, lon

def get_pos_by_json(url='geo_info/geoinfo-all.json'):
    """
    get all cities position form json file
    :return: dictionary of [cityName, (lat, lon)]
    """
    with open(url, 'r', encoding='UTF-8') as f:
        all_city_pos = json.loads(f.read())
    pos_dic = {}
    for province in all_city_pos:
        # pos_dic.update({province['name'].replace('市', ''): province['center']})
        pos_dic.update({province['name']: province['center']})
        if province['name'] != '重庆市':
            for city in province['districts']:
                pos_dic.update({city['name'].replace('市', ''): city['center']})
        else:
            for city in province['districts']:
                for districts in city['districts']:
                    pos_dic.update({districts['name']: districts['center']})
    return pos_dic

def get_cities_data(json_output, pos_dic):
    """
    get all cities data form json and position dictionary
    :return: list of [lat, lon, cityName, counts]
    """
    city_list = []
    for province in json_output:
        if province['provinceName'] == '台湾':
            lat, lon = get_lat_by_name(province['provinceName'], pos_dic)
            city_list.append([lat, lon, province['provinceName'], province['confirmedCount']])
        for city in province['cities']:
            try:
                lat, lon = get_lat_by_name(city['cityName'], pos_dic)
                city_list.append([lat, lon, city['cityName'], city['confirmedCount']])
            except KeyError:
                try:
                    lat, lon = get_lat_by_name(city['cityName' + '区'], pos_dic)
                    city_list.append([lat, lon, city['cityName'], city['confirmedCount']])
                except KeyError:
                    pass

    return city_list

# initial view state ：
view_state = pdk.ViewState(latitude=view_point[0], longitude=view_point[1], zoom=4, pitch=40, bearing=-27.36)

column_layer = pdk.Layer(
    "ColumnLayer",
    data=city_data,
    get_position=["lat", "lon"],
    radius=15000,
    diskResolution=24,
    elevation_scale=90,
    elevation_range=[0, 10000],
    getElevation='counts',
    getFillColor='[48, 128, counts*3, 225]',
    getLineColor=[0, 0, 0],
    pickable=True,
    )

st.write(pdk.Deck(
    map_style="mapbox://styles/mapbox/dark-v10",
    initial_view_state=view_state,
    layers=[column_layer],
))