import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

sns.set(color_codes=True)

tips = sns.load_dataset("tips")

#先看一个最简单的例子
#默认置信区间为95%

sns.regplot(x='total_bill',y='tip',data=tips)

sns.lmplot(x='total_bill',y='tip',data=tips)

sns.lmplot(x='size',y='tip',data=tips)

anscombe = sns.load_dataset("anscombe")
anscombe.head()

sns.lmplot(x='x',y='y',data=anscombe.query("dataset == 'I'"),ci=None,scatter_kws={"s": 80})

sns.lmplot(x="x", y="y", data=anscombe.query("dataset == 'II'"),
           ci=None, scatter_kws={"s": 80})

#上一个明明显拟合不到位，我们可以猜测这是一个多项式回归，利用order参数调用numpy.polyfit()
sns.lmplot(x="x", y="y", data=anscombe.query("dataset == 'II'"),order = 2 ,
           ci=None, scatter_kws={"s": 80})

sns.lmplot(x="x", y="y", data=anscombe.query("dataset == 'III'"),
           ci=None, scatter_kws={"s": 80});

## 存在异常值

# 不同的损失函数来减轻相对较大的残差
sns.lmplot(x="x", y="y", data=anscombe.query("dataset == 'III'"),
           robust=True, ci=None, scatter_kws={"s": 80});

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
%matplotlib inline

#去除kde
sns.distplot(x,kde=False,rug=True)

#箱子划分有多细，Seaborn会默认猜测一个，但是更好的应该由我们来指定
sns.distplot(x,bins=20,kde=False,rug=True)

sns.distplot(x,hist=False,rug=True)

sns.kdeplot(x,shade=True)

mean, cov = [0, 1], [(1, .5), (.5, 1)]
data = np.random.multivariate_normal(mean, cov, 200)
df = pd.DataFrame(data, columns=["x", "y"])
df.head()

sns.jointplot(x='x',y='y',data=df)

x, y = np.random.multivariate_normal(mean, cov, 1000).T
with sns.axes_style("white"):
    sns.jointplot(x=x, y=y, kind="hex", color="k")

sns.jointplot(x='x',y='y',kind='kde',data=df)

iris = sns.load_dataset("iris")
sns.pairplot(iris);

g = sns.PairGrid(iris)
g.map_diag(sns.kdeplot)
g.map_offdiag(sns.kdeplot, n_levels=6);

import seaborn as sns
import matplotlib.pyplot as plt
sns.set(style="ticks", color_codes=True)

tips = sns.load_dataset("tips")
tips.head()

sns.catplot(x='day',y='total_bill',data=tips)

#The jitter parameter controls the magnitude of jitter or disables it altogether:
sns.catplot(x="day", y="total_bill", jitter=False, data=tips);

sns.catplot(x="day", y="total_bill", kind="swarm", data=tips);

#也支持hue参数进行分类，但是不支持style
sns.catplot(x="day", y="total_bill", kind="swarm",hue='sex',data=tips);

#也可以在轴上对一个参数进行分类,参数order
sns.catplot(x='smoker',y='tip',order=['Yes','No'],data=tips)

# x,y轴是很自由的，换一种展现方式
sns.catplot(x='tip',y='smoker',order=['Yes','No'],data=tips)

sns.catplot(x='day',y='total_bill',kind='box',data=tips)

sns.catplot(x='day',y='total_bill',kind='box',hue='sex',data=tips)

sns.catplot(x='day',y='total_bill',kind='boxen',hue='sex',data=tips)

sns.catplot(x='total_bill',y='day',kind='violin',data=tips)

sns.catplot(x="day", y="total_bill", hue="sex",
            kind="violin", split=True, data=tips);

titanic = sns.load_dataset("titanic")
titanic.head()

sns.catplot(x='sex',y='survived',kind='bar',hue='class',data=titanic)

sns.catplot(x='deck',kind='count',palette='ch:.25',data=titanic)

sns.catplot(x='sex',y='survived',kind='point',hue='class',data=titanic)

#当然也可以标记得更好,刻画palette,markers,linestyles等参数
sns.catplot(x='class',y='survived',kind='point',hue='sex',data=titanic
           ,palette={'male':'g','female':'m'}
           ,markers=['^','o'],linestyles=['-','--'])

iris = sns.load_dataset("iris")
sns.catplot(data=iris, orient="h", kind="box")

sns.catplot(x="day", y="total_bill", hue="smoker",
            col="time", aspect=.6,
            kind="swarm", data=tips);

g = sns.catplot(x="fare", y="survived", row="class",
                kind="box", orient="h", height=1.5, aspect=4,
                data=titanic.query("fare > 0"))
g.set(xscale="log");

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
sns.set(style="darkgrid")

tips = sns.load_dataset("tips")   # load_dateset()是从在线存储库加载数据集，极大方便了练习
tips.head()

sns.relplot(x="total_bill", y="tip", data=tips)

sns.relplot(x="total_bill", y="tip",hue='size', data=tips) #hue参数对输入的变量进行分组，生成的不同的颜色

sns.relplot(x="total_bill", y="tip",hue='size',size='size',sizes=(15,100),data=tips)  # size,sizes一般搭配使用

sns.relplot(x="total_bill", y="tip",hue='size',size='size',sizes=(15,50),col='time',data=tips)

sns.relplot(x="total_bill", y="tip",hue='size',size='size',sizes=(15,50),col='time',row='sex',data=tips)

sns.relplot(x="total_bill", y="tip",hue='time',col ='time',palette = ['b','r'],data=tips) #指定绘图的颜色

#lineplot()函数的很多参数其实和replot()一致，hue|size|col之类的，可以参考

df = pd.DataFrame(dict(time=np.arange(500),value=np.random.randn(500).cumsum()))
sns.relplot(x='time',y='value',kind='line',data=df)

sns.lineplot(x='time',y='value',data=df)

fmri = sns.load_dataset("fmri")
sns.relplot(x="timepoint", y="signal", kind="line", data=fmri)

#特别是对于较大的数据，可以通过绘制标准偏差而不是置信区间来表示每个时间点的分布扩散
sns.relplot(x="timepoint", y="signal", kind="line",ci='sd',data=fmri)

fmri.head()

sns.relplot(x="timepoint", y="signal", hue="event", kind="line", data=fmri);

sns.lineplot(x='timepoint',y='signal',hue='region',style='event',data=fmri)

#标识子集
sns.relplot(x="timepoint", y="signal", hue="region", style="event",
            dashes=False, markers=True, kind="line", marker =True,data=fmri);

sns.relplot(x="timepoint", y="signal", hue="region",
            units="subject", estimator=None,
            kind="line", data=fmri.query("event == 'stim'"))

dots = sns.load_dataset("dots").query("align == 'dots'")
sns.relplot(x="time", y="firing_rate",
            hue="coherence", style="choice",
            kind="line", data=dots)

df = pd.DataFrame(dict(time=pd.date_range("2017-1-1", periods=500),
                       value=np.random.randn(500).cumsum()))
g = sns.relplot(x="time", y="value", kind="line", data=df)
g.fig.autofmt_xdate()

sns.relplot(x="timepoint", y="signal", hue="subject",
            col="region", row="event", height=3,
            kind="line", estimator=None, data=fmri);

sns.relplot(x="timepoint", y="signal", hue="event", style="event",
            col="subject", col_wrap=5,
            height=3, aspect=.75, linewidth=2.5,
            kind="line", data=fmri.query("region == 'frontal'"));

import numpy as np
import pandas as pd
PREVIOUS_MAX_ROWS = pd.options.display.max_rows
pd.options.display.max_rows = 20
np.random.seed(12345)
import matplotlib.pyplot as plt
import matplotlib
plt.rc('figure', figsize=(10, 6))
np.set_printoptions(precision=4, suppress=True)

plt.plot(np.arange(10))

fig = plt.figure()
ax1 = fig.add_subplot(2,2,1)
ax2 = fig.add_subplot(2,2,2)
ax3 = fig.add_subplot(2,2,3)

plt.plot(np.random.randn(50).cumsum(),'k--')

_ = ax1.hist(np.random.randn(100),bins=20,color='k',alpha=0.3)
ax2.scatter(np.arange(30), np.arange(30) + 3 * np.random.randn(30))

fig

fig, axes = plt.subplots(2, 3)
axes

fig, axes = plt.subplots(2, 2, sharex=True, sharey=True)
for i in range(2):
    for j in range(2):
        axes[i,j].hist(np.random.randn(500), bins=50, color='k', alpha=0.5)
plt.subplots_adjust(wspace=0.1,hspace=0.1)

plt.figure()

from numpy.random import randn
plt.plot(randn(50).cumsum(),linestyle='--',color='b')

plt.plot(randn(30).cumsum(), color='k', linestyle='dashed', marker='o')

plt.plot(randn(30).cumsum(),'ko--')   #ko--是把参数组合在一起了.... color = 'k' marker = 'o' linestyle = '--'

plt.close('all')

data = np.random.randn(30).cumsum()
plt.plot(data, 'k--', label='Default')
plt.plot(data, 'k-', drawstyle='steps-post', label='steps-post')
plt.legend(loc='best')

fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(randn(1000).cumsum())

ticks  = ax.set_xticks([0,250,500,750,1000])
labels = ax.set_xticklabels(['one', 'two', 'three', 'four', 'five'],
                            rotation=30, fontsize='small')

ax.set_title('My First Title of Matplotlib')

ax.set_xlabel('Stage')

fig

fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(randn(100).cumsum(), 'k', label='one')
ax.plot(randn(100).cumsum(), 'k--', label='two')
ax.plot(randn(100).cumsum(), 'k.', label='three')

ax.legend(loc='best')
fig

from datetime import datetime

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)

data = pd.read_csv('examples/spx.csv', index_col=0, parse_dates=True)
spx = data['SPX']

spx.plot(ax=ax, style='k-')

crisis_data = [
    (datetime(2007, 10, 11), 'Peak of bull market'),
    (datetime(2008, 3, 12), 'Bear Stearns Fails'),
    (datetime(2008, 9, 15), 'Lehman Bankruptcy')
]

for date, label in crisis_data:
    ax.annotate(label, xy=(date, spx.asof(date) + 75),
                xytext=(date, spx.asof(date) + 225),
                arrowprops=dict(facecolor='black', headwidth=4, width=2,
                                headlength=4),
                horizontalalignment='left', verticalalignment='top')

# Zoom in on 2007-2010
ax.set_xlim(['1/1/2007', '1/1/2011'])
ax.set_ylim([600, 1800])

ax.set_title('Important dates in the 2008-2009 financial crisis')

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)

rect = plt.Rectangle((0.2, 0.75), 0.4, 0.15, color='k', alpha=0.3)
circ = plt.Circle((0.7, 0.2), 0.15, color='b', alpha=0.3)
pgon = plt.Polygon([[0.15, 0.15], [0.35, 0.4], [0.2, 0.6]],
                   color='g', alpha=0.5)

ax.add_patch(rect)
ax.add_patch(circ)
ax.add_patch(pgon)

fig.savefig('/Users/zhangyangfenbi.com/Desktop/demo.png')

import numpy as np
import pandas as pd
pd.options.display.max_rows = 20
np.random.seed(12345)
import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))
np.set_printoptions(precision=4, suppress=True)

data = pd.Series(np.random.randn(9),
                 index=[['a', 'a', 'a', 'b', 'b', 'c', 'c', 'd', 'd'],
                        [1, 2, 3, 1, 3, 1, 2, 2, 3]])
data

data.index

data['b']

data['b':'c']

data.loc[['b', 'd']]

data.loc[:, 2]

frame = pd.DataFrame(np.arange(12).reshape((4, 3)),
                     index=[['a', 'a', 'b', 'b'], [1, 2, 1, 2]],
                     columns=[['Ohio', 'Ohio', 'Colorado'],
                              ['Green', 'Red', 'Green']])
frame

frame.index.names = ['key1', 'key2']
frame.columns.names = ['state', 'color']
frame

frame['Ohio']

df1 = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'a', 'b'],
                    'data1': range(7)})
df2 = pd.DataFrame({'key': ['a', 'b', 'd'],
                    'data2': range(3)})

df1

df2

pd.merge(df1,df2)  #如果没有指定建，则会默认将重叠列名做键

df3 = pd.DataFrame({'lkey': ['b', 'b', 'a', 'c', 'a', 'a', 'b'],
                    'data1': range(7)})
df4 = pd.DataFrame({'rkey': ['a', 'b', 'd'],
                    'data2': range(3)})
print(df3)
print(df4)

pd.merge(df3,df4,left_on='lkey',right_on='rkey') #默认Inner连接，how=''参数决定怎么连接

#左连接
pd.merge(df3,df4,left_on='lkey',right_on='rkey',how='left')

## 多个键进行连接时，传入列表就好
left = pd.DataFrame({'key1': ['foo', 'foo', 'bar'],
                     'key2': ['one', 'two', 'one'],
                     'lval': [1, 2, 3]})
right = pd.DataFrame({'key1': ['foo', 'foo', 'bar', 'bar'],
                      'key2': ['one', 'one', 'one', 'two'],
                      'rval': [4, 5, 6, 7]})
pd.merge(left,right,on=['key1','key2'],how='left')

pd.merge(left, right, on='key1')

pd.merge(left, right, on='key1', suffixes=('_left', '_right'))

left1 = pd.DataFrame({'key': ['a', 'b', 'a', 'a', 'b', 'c'],
                      'value': range(6)})
right1 = pd.DataFrame({'group_val': [3.5, 7]}, index=['a', 'b'])

left1

right1

pd.merge(left1,right1,left_on='key',right_index=True)

lefth = pd.DataFrame({'key1': ['Ohio', 'Ohio', 'Ohio',
                               'Nevada', 'Nevada'],
                      'key2': [2000, 2001, 2002, 2001, 2002],
                      'data': np.arange(5.)})
righth = pd.DataFrame(np.arange(12).reshape((6, 2)),
                      index=[['Nevada', 'Nevada', 'Ohio', 'Ohio',
                              'Ohio', 'Ohio'],
                             [2001, 2000, 2000, 2000, 2001, 2002]],
                      columns=['event1', 'event2'])

lefth

righth

pd.merge(lefth,righth,left_on=['key1','key2'],right_index=True)

pd.merge(lefth,righth,left_on=['key1','key2'],right_index=True,how='outer')

##两边都开启索引也没有问题

left2 = pd.DataFrame([[1., 2.], [3., 4.], [5., 6.]],
                     index=['a', 'c', 'e'],
                     columns=['Ohio', 'Nevada'])
right2 = pd.DataFrame([[7., 8.], [9., 10.], [11., 12.], [13, 14]],
                      index=['b', 'c', 'd', 'e'],
                      columns=['Missouri', 'Alabama'])

pd.merge(left2, right2, how='outer', left_index=True, right_index=True)

left2.join(right2)    ###更加快速地实现索引合并

left1.join(right1, on='key')

arr=np.arange(12).reshape(3,4)
arr

np.concatenate([arr, arr], axis=1)

s1 = pd.Series([0, 1], index=['a', 'b'])
s2 = pd.Series([2, 3, 4], index=['c', 'd', 'e'])
s3 = pd.Series([5, 6], index=['f', 'g'])

pd.concat([s1,s2,s3]) #默认axis=0

s4=pd.concat([s1*5,s3])
pd.concat([s1, s4], axis=1, join_axes=[['a', 'c', 'b', 'e']])

result = pd.concat([s1,s2,s3],keys=['one','two','three'])
result

s1

result.unstack()

pd.concat([s1, s2, s3], axis=1, keys=['one', 'two', 'three'],sort=True)

df1 = pd.DataFrame(np.arange(6).reshape(3, 2), index=['a', 'b', 'c'],
                   columns=['one', 'two'])
df2 = pd.DataFrame(5 + np.arange(4).reshape(2, 2), index=['a', 'c'],
                   columns=['three', 'four'])
df1

df2

pd.concat([df1, df2], axis=1,sort=True)

pd.concat({'level1': df1, 'level2': df2}, axis=1,sort=True) #字典的键会被当做keys选项的值

pd.concat([df1,df2],axis=1,keys=['l1','l2'],names=['zz1','zz2'],sort=True)

df1 = pd.DataFrame(np.random.randn(3, 4), columns=['a', 'b', 'c', 'd'])
df2 = pd.DataFrame(np.random.randn(2, 3), columns=['b', 'd', 'a'])

df1

df2

pd.concat([df1,df2],sort=True)

pd.concat([df1,df2],ignore_index=True,sort=True)

a = pd.Series([np.nan, 2.5, np.nan, 3.5, 4.5, np.nan],
              index=['f', 'e', 'd', 'c', 'b', 'a'])
b = pd.Series(np.arange(len(a), dtype=np.float64),
              index=['f', 'e', 'd', 'c', 'b', 'a'])
b[-1]=np.nan

a

b

np.where(pd.isnull(a),b,a) #用b对应索引的值来填充a的空值

b[:-2].combine_first(a[2:])

df1 = pd.DataFrame({'a': [1., np.nan, 5., np.nan],
                    'b': [np.nan, 2., np.nan, 6.],
                    'c': range(2, 18, 4)})
df2 = pd.DataFrame({'a': [5., 4., np.nan, 3., 7.],
                    'b': [np.nan, 3., 4., 6., 8.]})

df1

df2

df1.combine_first(df2) #用参数对象的数据为调用者对象的缺失数据‘打补丁’

data = pd.DataFrame(np.arange(6).reshape((2, 3)),
                    index=pd.Index(['Ohio', 'Colorado'], name='state'),
                    columns=pd.Index(['one', 'two', 'three'],
                    name='number'))
data

result = data.stack()
result

result.unstack()

data = pd.read_csv('examples/macrodata.csv')
data.head()
periods = pd.PeriodIndex(year=data.year, quarter=data.quarter,
                         name='date')
columns = pd.Index(['realgdp', 'infl', 'unemp'], name='item')
data = data.reindex(columns=columns)
data.index = periods.to_timestamp('D', 'end')
ldata = data.stack().reset_index().rename(columns={0: 'value'})

ldata[:10]

pivoted = ldata.pivot('date','item','value')  ## index , columns , value 
pivoted.head()

ldata['value2'] = np.random.randn(len(ldata))

ldata[:10]

pivoted = ldata.pivot('date','item')
pivoted.head()   # 带有层次化索引的列

pivoted['value'].head()

data=pd.DataFrame({'k1':['one']*3+['two']*4,'k2':[1,1,2,3,3,4,4]})
data

data.duplicated() #判断当前行是否是重复行

data.drop_duplicates() #移除重复行

data['v1']=np.arange(7)
data.drop_duplicates(['k1'])  #只根据某一行来移除

data=pd.Series([1,2,3,-99])
data

data.replace(-99,np.nan)

data.replace({1:100,-99:np.nan})

data = pd.DataFrame([[7., 8.], [9., 10.], [11., 12.], [13, 14]],
                      index=['b', 'c', 'd', 'e'],
                      columns=['Missouri', 'Alabama'])
data

data.index.map(str.upper)

#离散化
age = np.arange(10)
bins = [2,5,8]
cats = pd.cut(age,bins)
cats

from pandas import DataFrame
data=DataFrame(np.random.randn(1000,4))
data.describe()

#找出某列中绝对值超过3的值
col=data[3]
col[abs(col)>3]

data[(np.abs(data)>3).any(1)][:2]

data[(np.abs(data)>3).any(1)]=np.sign(data)*3

df = DataFrame({'key':['b','b','a','c','a','b'],'value':np.arange(6)})
df

pd.get_dummies(df['key'])

#和Python编码一起学习下

import re
#参考之前的博客

import numpy as np
import pandas as pd
np.random.seed(12345)
import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))
np.set_printoptions(precision=4, suppress=True)

!pwd

!cat examples/ex1.csv

file_path = 'examples/ex1.csv'
df=pd.read_csv(file_path)
df

#read_table也可以，不过分隔符不一样，需要重新指定
pd.read_table('examples/ex1.csv',sep=',')

!cat examples/ex2.csv

#可以让pandas为其默认分配列名，或者自己定义列名
df1=pd.read_csv('examples/ex2.csv',header=None)
df1

df2=pd.read_csv('examples/ex2.csv',names=['a','b','c','d','e'])
df2

df2=pd.read_csv('examples/ex2.csv',names=['a','b','c','d','e'],index_col='e')
df2

!cat examples/csv_mindex.csv

pd.read_csv('examples/csv_mindex.csv',index_col=['key1','key2'])

#
list(open('examples/ex3.txt'))

#通过正则表达式去匹配并不是固定的分隔符
res = pd.read_table('examples/ex3.txt',sep='\s+')
res

!cat examples/ex4.csv

#使用skiprows跳过指定行
pd.read_csv('examples/ex4.csv',skiprows=[0,2,3])

!cat examples/ex5.csv

#pandas会对缺失值进行标识
res1=pd.read_csv('examples/ex5.csv')
res1

res1.isnull()

result = pd.read_csv('examples/ex5.csv', na_values=['NULL'])
result

chunker = pd.read_csv('examples/ex6.csv', chunksize=1000)
chunker

chunker = pd.read_csv('examples/ex6.csv', chunksize=1000)

tot = pd.Series([])
for piece in chunker:
    tot = tot.add(piece['key'].value_counts(), fill_value=0)

tot = tot.sort_values(ascending=False)

# Top5
tot[:5]

data = pd.read_csv('examples/ex5.csv')
data

pwd

data.to_csv('/Users/zhangyangfenbi.com/Desktop/tmp.csv')

!cat '/Users/zhangyangfenbi.com/Desktop/tmp.csv'

import sys
data.to_csv(sys.stdout, sep='|')

data.to_csv(sys.stdout, index=False, header=False)

data.to_csv(sys.stdout, index=False, columns=['a', 'b', 'c'])

dates = pd.date_range('1/1/2000', periods=7)
ts = pd.Series(np.arange(7), index=dates)
ts.to_csv('examples/tseries.csv')
!cat examples/tseries.csv

import csv
f = open('examples/ex7.csv')

reader = csv.reader(f)
for line in reader:
    print(line)
    
#writer省略

#Json格式已经成为一种通用的格式，主要用户http请求和应用程序之间发送数据
obj = """
{"name": "Wes",
 "places_lived": ["United States", "Spain", "Germany"],
 "pet": null,
 "siblings": [{"name": "Scott", "age": 30, "pets": ["Zeus", "Zuko"]},
              {"name": "Katie", "age": 38,
               "pets": ["Sixes", "Stache", "Cisco"]}]
}
"""