/img/programming/2022/04/apache_nifi_installation_windows/apache_nifi_installation_windows
$ sudo apt install python3-pip
[sudo] password for username:
$ sudo pip3 install virtualenv
$ virtualenv venv
$ source venv/bin/activate
$ vi ~/.bashrc
export AIRFLOW_HOME=/mnt/c/airflow-test
$ source ~/.bashrc
echo $AIRFLOW_HOME
/mnt/c/airflow-test
$ pip3 install 'apache-airflow[postgres, slack, celery]'
$ airflow db init
$ airflow webserver -p 8081
$ airflow scheduler
$ dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart
$ dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart
$ wsl --set-default-version 2
WSL 2와의 주요 차이점에 대한 자세한 내용은 https://aka.ms/wsl2를 참조하세요
스타벅스 아이스 아메리카노를 선물로 보내드리겠습니다.
import sklearn
print("sklearn ver.", sklearn.__version__)
sklearn ver. 1.0.2
from sklearn.preprocessing import OneHotEncoder
enc = OneHotEncoder(handle_unknown='ignore')
X = [['Male', 1], ['Female', 3], ['Female', 2]]
enc.fit_transform(X).toarray()
array([[0., 1., 1., 0., 0.],
[1., 0., 0., 0., 1.],
[1., 0., 0., 1., 0.]])
from sklearn.preprocessing import OneHotEncoder
from seaborn import load_dataset
penguins = load_dataset('penguins')
ohe = OneHotEncoder()
transformed = ohe.fit_transform(penguins[['island']])
print(transformed.toarray())
print(ohe.categories_)
print(penguins['island'].unique())
[[0. 0. 1.]
[0. 0. 1.]
[0. 0. 1.]
...
[1. 0. 0.]
[1. 0. 0.]
[1. 0. 0.]]
[array(['Biscoe', 'Dream', 'Torgersen'], dtype=object)]
['Torgersen' 'Biscoe' 'Dream']
print(penguins.head())
species island bill_length_mm bill_depth_mm flipper_length_mm \
0 Adelie Torgersen 39.1 18.7 181.0
1 Adelie Torgersen 39.5 17.4 186.0
2 Adelie Torgersen 40.3 18.0 195.0
3 Adelie Torgersen NaN NaN NaN
4 Adelie Torgersen 36.7 19.3 193.0
body_mass_g sex
0 3750.0 Male
1 3800.0 Female
2 3250.0 Female
3 NaN NaN
4 3450.0 Female
penguins[ohe.categories_[0]] = transformed.toarray()
print(penguins.head())
species island bill_length_mm bill_depth_mm flipper_length_mm \
0 Adelie Torgersen 39.1 18.7 181.0
1 Adelie Torgersen 39.5 17.4 186.0
2 Adelie Torgersen 40.3 18.0 195.0
3 Adelie Torgersen NaN NaN NaN
4 Adelie Torgersen 36.7 19.3 193.0
body_mass_g sex Biscoe Dream Torgersen
0 3750.0 Male 0.0 0.0 1.0
1 3800.0 Female 0.0 0.0 1.0
2 3250.0 Female 0.0 0.0 1.0
3 NaN NaN 0.0 0.0 1.0
4 3450.0 Female 0.0 0.0 1.0
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.compose import make_column_transformer
from seaborn import load_dataset
import pandas as pd
penguins = load_dataset('penguins')
sample_cols = ['island', 'sex', 'bill_length_mm', 'species']
penguins = penguins[sample_cols]
# 결측치 제거
penguins = penguins.dropna()
print(penguins.head())
print(penguins.info())
island sex bill_length_mm species
0 Torgersen Male 39.1 Adelie
1 Torgersen Female 39.5 Adelie
2 Torgersen Female 40.3 Adelie
4 Torgersen Female 36.7 Adelie
5 Torgersen Male 39.3 Adelie
<class 'pandas.core.frame.DataFrame'>
Int64Index: 333 entries, 0 to 343
Data columns (total 4 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 island 333 non-null object
1 sex 333 non-null object
2 bill_length_mm 333 non-null float64
3 species 333 non-null object
dtypes: float64(1), object(3)
memory usage: 13.0+ KB
None
categorical_cols = ['island', 'sex']
label_cols = ['species']
transformer = make_column_transformer(
(OneHotEncoder(), categorical_cols),
remainder='passthrough',
verbose_feature_names_out = False)
transformed = transformer.fit_transform(penguins)
transformed_df = pd.DataFrame(transformed, columns=transformer.get_feature_names_out())
print(transformed_df.head())
island_Biscoe island_Dream island_Torgersen sex_Female sex_Male \
0 0.0 0.0 1.0 0.0 1.0
1 0.0 0.0 1.0 1.0 0.0
2 0.0 0.0 1.0 1.0 0.0
3 0.0 0.0 1.0 1.0 0.0
4 0.0 0.0 1.0 0.0 1.0
bill_length_mm species
0 39.1 Adelie
1 39.5 Adelie
2 40.3 Adelie
3 36.7 Adelie
4 39.3 Adelie
import pandas as pd
from seaborn import load_dataset
tips = load_dataset('tips')
# 결측치 제거
tips = tips.dropna()
print(tips.info())
<class 'pandas.core.frame.DataFrame'>
Int64Index: 244 entries, 0 to 243
Data columns (total 7 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 total_bill 244 non-null float64
1 tip 244 non-null float64
2 sex 244 non-null category
3 smoker 244 non-null category
4 day 244 non-null category
5 time 244 non-null category
6 size 244 non-null int64
dtypes: category(4), float64(2), int64(1)
memory usage: 9.1 KB
None
import warnings
import sklearn
import pandas as pd
import numpy as np
def get_feature_names(column_transformer):
"""Get feature names from all transformers.
Returns
-------
feature_names : list of strings
Names of the features produced by transform.
"""
# Remove the internal helper function
#check_is_fitted(column_transformer)
# Turn loopkup into function for better handling with pipeline later
def get_names(trans):
# >> Original get_feature_names() method
if trans == 'drop' or (
hasattr(column, '__len__') and not len(column)):
return []
if trans == 'passthrough':
if hasattr(column_transformer, '_df_columns'):
if ((not isinstance(column, slice))
and all(isinstance(col, str) for col in column)):
return column
else:
return column_transformer._df_columns[column]
else:
indices = np.arange(column_transformer._n_features)
return ['x%d' % i for i in indices[column]]
if not hasattr(trans, 'get_feature_names'):
# >>> Change: Return input column names if no method avaiable
# Turn error into a warning
warnings.warn("Transformer %s (type %s) does not "
"provide get_feature_names. "
"Will return input column names if available"
% (str(name), type(trans).__name__))
# For transformers without a get_features_names method, use the input
# names to the column transformer
if column is None:
return []
else:
return [name + "__" + f for f in column]
return [name + "__" + f for f in trans.get_feature_names()]
### Start of processing
feature_names = []
# Allow transformers to be pipelines. Pipeline steps are named differently, so preprocessing is needed
if type(column_transformer) == sklearn.pipeline.Pipeline:
l_transformers = [(name, trans, None, None) for step, name, trans in column_transformer._iter()]
else:
# For column transformers, follow the original method
l_transformers = list(column_transformer._iter(fitted=True))
for name, trans, column, _ in l_transformers:
if type(trans) == sklearn.pipeline.Pipeline:
# Recursive call on pipeline
_names = get_feature_names(trans)
# if pipeline has no transformer that returns names
if len(_names)==0:
_names = [name + "__" + f for f in column]
feature_names.extend(_names)
else:
feature_names.extend(get_names(trans))
return feature_names
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
categorical_cols = ['sex', 'day']
ordinal_cols = ['smoker', 'time']
numeric_cols = ['total_bill']
keep_features = [x for x in tips.columns if x not in categorical_cols + ordinal_cols + numeric_cols]
tips2 = tips[categorical_cols + ordinal_cols + numeric_cols]
transformer = ColumnTransformer(
[('StandardScaler', StandardScaler(), numeric_cols),
('OneHotEncoder', OneHotEncoder(), categorical_cols),
('OrdinalEncoder', OrdinalEncoder(), ordinal_cols)],
remainder='passthrough',
verbose_feature_names_out = False)
transformed = transformer.fit_transform(tips2)
transformed_df = pd.DataFrame(transformed, columns=get_feature_names(transformer))
tip3 = pd.concat([tips[keep_features], transformed_df], axis = 1)
tip3.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 244 entries, 0 to 243
Data columns (total 11 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 tip 244 non-null float64
1 size 244 non-null int64
2 StandardScaler__total_bill 244 non-null float64
3 OneHotEncoder__x0_Female 244 non-null float64
4 OneHotEncoder__x0_Male 244 non-null float64
5 OneHotEncoder__x1_Fri 244 non-null float64
6 OneHotEncoder__x1_Sat 244 non-null float64
7 OneHotEncoder__x1_Sun 244 non-null float64
8 OneHotEncoder__x1_Thur 244 non-null float64
9 OrdinalEncoder__smoker 244 non-null float64
10 OrdinalEncoder__time 244 non-null float64
dtypes: float64(10), int64(1)
memory usage: 22.9 KB
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: UserWarning: Transformer StandardScaler (type StandardScaler) does not provide get_feature_names. Will return input column names if available
/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.
warnings.warn(msg, category=FutureWarning)
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:38: UserWarning: Transformer OrdinalEncoder (type OrdinalEncoder) does not provide get_feature_names. Will return input column names if available
!pip install -U matplotlib
Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)
Collecting matplotlib
Downloading matplotlib-3.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.2 MB)
[K |████████████████████████████████| 11.2 MB 27.0 MB/s
[?25hRequirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.4.0)
Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.2)
Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.21.5)
Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (7.1.2)
Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (3.0.7)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.11.0)
Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (21.3)
Collecting fonttools>=4.22.0
Downloading fonttools-4.31.2-py3-none-any.whl (899 kB)
[K |████████████████████████████████| 899 kB 50.5 MB/s
[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib) (3.10.0.2)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7->matplotlib) (1.15.0)
Installing collected packages: fonttools, matplotlib
Attempting uninstall: matplotlib
Found existing installation: matplotlib 3.2.2
Uninstalling matplotlib-3.2.2:
Successfully uninstalled matplotlib-3.2.2
[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.[0m
Successfully installed fonttools-4.31.2 matplotlib-3.5.1
%matplotlib inline
import sklearn
print(sklearn.__version__)
import matplotlib
print(matplotlib.__version__)
# 필수 라이브러리 불러오기
from sklearn.datasets import load_iris
from sklearn import tree
import matplotlib.pyplot as plt
# 데이터 불러오기
iris = load_iris()
print(iris.data.shape, iris.target.shape)
print("feature names", iris.feature_names)
print("class names", iris.target_names)
# 모형 학습 및 plot_tree 그래프 구현
dt = tree.DecisionTreeClassifier(random_state=0)
dt.fit(iris.data, iris.target)
fig, ax = plt.subplots(figsize=(10, 6))
ax = tree.plot_tree(dt, max_depth = 2, filled=True, feature_names = iris.feature_names, class_names = iris.target_names)
plt.show()
1.0.2
3.5.1
(150, 4) (150,)
feature names ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']
class names ['setosa' 'versicolor' 'virginica']
> library(dplyr)
> library(readxl)
> raw_df = read_excel("data/NewsResult_20211213-20220313.xlsx", sheet = 1)
> raw_df2 = raw_df %>% select(일자, 언론사, 제목, URL)
> raw_df2 %>% group_by(언론사) %>% summarise(n = n())
# A tibble: 3 × 2
언론사 n
<chr> <int>
1 국민일보 180
2 조선일보 115
3 중앙일보 256
> kmib_df = raw_df2 %>% filter(언론사 == "국민일보")
> head(kmib_df, 3)
# A tibble: 3 × 4 일자 언론사 제목 URL
<chr> <chr> <chr> <chr>
1 20220312 국민일보 팬데믹의 비극 무너지는 가정, 스러지는 아이들 http://news.kmib.co…
2 20220312 국민일보 ‘장로’ 디딤돌인가 걸림돌인가 http://news.kmib.co…
3 20220311 국민일보 [기고]대전은 우리가 지킨다 http://news.kmib.co…
&cp=kd 같은 문자가 더 추가된 것을 확인할 수 있다.> kmib_df$URL[1:5]
[1] "http://news.kmib.co.kr/article/view.asp?arcid=0924235144&code=11131100"
[2] "http://news.kmib.co.kr/article/view.asp?arcid=0924235120&code=23111111"
[3] "http://news.kmib.co.kr/article/view.asp?arcid=0016856942&code=61221514&cp=kd"
[4] "http://news.kmib.co.kr/article/view.asp?arcid=0016853803&code=61111711&cp=kd"
[5] "http://news.kmib.co.kr/article/view.asp?arcid=0016847353&code=61131111&cp=kd"
> url = kmib_df$URL[1]
> news = read_html(url, encoding = "EUC-KR")
> news
{html_document}<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="ko">
[1] <head>\n<title>팬데믹의 비극… 무너지는 가정, 스러지는 아이들-국민일보</title>\n<meta http-equiv="Cont ...
[2] <body>\r\n<div id="wrap">\r\n\r\n<!-- header -->\r\n\r\n<script src="http://ww ...
Procfile 을 생성한다.web: gunicorn index:server
index 파일명을 의미한다.server = app.server 을 추가한다.app = dash.Dash(__name__, meta_tags=[{"name": "viewport",
"content": "width=device-width"}])
server = app.server
runtime.txt)
python-3.8.7
Heroku 회원가입을 안했다면, 진행한다.
$ git clone https://github.com/your_id/python_dash_sales.git
PyCharm을 주 에디터로 사용할 예정이다.
ML.EVALUATE 함수를 사용하여 ML 모델 평가ML.PREDICT 함수를 사용하여 ML 모델 예측bqml_practice 입력
species — 펭귄의 종(문자열)island — 펭귄이 사는 섬(문자열)culmen_length_mm — 컬멘 길이(밀리미터)(FLOAT64).culmen_depth_mm — 컬멘 깊이(밀리미터)(FLOAT64)flipper_length_mm — 지느러미의 길이(밀리미터)(FLOAT64)sex — 펭귄의 성별(문자열)CREATE MODEL 명령어를 실행하여 모델을 생성한다.#standardSQL
CREATE OR REPLACE MODEL `bqml_practice.penguins_model`
OPTIONS
(model_type='linear_reg',
input_label_cols=['body_mass_g']) AS
SELECT
*
FROM
`bigquery-public-data.ml_datasets.penguins`
WHERE
body_mass_g IS NOT NULL
TRY IT FREE 를 클릭한다.