import plotly.express as px

fig = px.line(x=["a","b","c"], y=[1,3,2], title="sample figure")
fig.show()

$ sudo apt update
$ sudo apt install apt-transport-https

$ sudo apt install openjdk-11-jdk

$ java -version
openjdk version "11.0.14.1" 2022-02-08
OpenJDK Runtime Environment (build 11.0.14.1+1-Ubuntu-0ubuntu1.20.04)
OpenJDK 64-Bit Server VM (build 11.0.14.1+1-Ubuntu-0ubuntu1.20.04, mixed mode, sharing)

$ sudo vi /etc/environment

$ source /etc/environment

$ echo $JAVA_HOME
/usr/lib/jvm/java-11-openjdk-amd64

$ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
OK

$ sudo sh -c 'echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" > /etc/apt/sources.list.d/elastic-7.x.list'

$ sudo apt-get update
Hit:1 https://artifacts.elastic.co/packages/7.x/apt stable InRelease
Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease
Hit:3 http://archive.ubuntu.com/ubuntu focal InRelease
Hit:4 http://archive.ubuntu.com/ubuntu focal-updates InRelease
Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease
Reading package lists... Done
$ sudo apt-get install elasticsearch
Reading package lists... Done
Building dependency tree
Reading state information... Done
The following NEW packages will be installed:
  elasticsearch
0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
Need to get 312 MB of archives.
After this operation, 517 MB of additional disk space will be used.
Get:1 https://artifacts.elastic.co/packages/7.x/apt stable/main amd64 elasticsearch amd64 7.17.2 [312 MB]
Fetched 312 MB in 8s (40.9 MB/s)
Selecting previously unselected package elasticsearch.
(Reading database ... 32942 files and directories currently installed.)
Preparing to unpack .../elasticsearch_7.17.2_amd64.deb ...
Creating elasticsearch group... OK
Creating elasticsearch user... OK
Unpacking elasticsearch (7.17.2) ...
Setting up elasticsearch (7.17.2) ...
### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd
 sudo systemctl daemon-reload
 sudo systemctl enable elasticsearch.service
### You can start elasticsearch service by executing
 sudo systemctl start elasticsearch.service
warning: usage of JAVA_HOME is deprecated, use ES_JAVA_HOME
Created elasticsearch keystore in /etc/elasticsearch/elasticsearch.keystore
Processing triggers for systemd (245.4-4ubuntu3.15) ...

$ sudo systemctl start elasticsearch

$ sudo -b unshare --pid --fork --mount-proc /lib/systemd/systemd --system-unit=basic.target
$ sudo -E nsenter --all -t $(pgrep -xo systemd) runuser -P -l $USER -c "exec $SHELL"

$ sudo systemctl enable elasticsearch
Synchronizing state of elasticsearch.service with SysV service script with /lib/systemd/systemd-sysv-install.
Executing: /lib/systemd/systemd-sysv-install enable elasticsearch
Created symlink /etc/systemd/system/multi-user.target.wants/elasticsearch.service → /lib/systemd/system/elasticsearch.service.

$ sudo systemctl start elasticsearch

$ curl -X GET "localhost:9200/"
{
  "name" : "evan",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "vATWEOO1T9yOFGLc7G3L4w",
  "version" : {
    "number" : "7.17.2",
    "build_flavor" : "default",
    "build_type" : "deb",
    "build_hash" : "de7261de50d90919ae53b0eff9413fd7e5307301",
    "build_date" : "2022-03-28T15:12:21.446567561Z",
    "build_snapshot" : false,
    "lucene_version" : "8.11.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

$ sudo apt update
[sudo] password for evan:
Hit:1 https://artifacts.elastic.co/packages/7.x/apt stable InRelease
Get:2 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB]
Hit:3 http://archive.ubuntu.com/ubuntu focal InRelease
Get:4 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB]
Get:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB]
Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [1712 kB]
Get:7 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [916 kB]
Fetched 2963 kB in 5s (600 kB/s)
Reading package lists... Done
Building dependency tree
Reading state information... Done
All packages are up to date.

$ sudo apt-get upgrade
Reading package lists... Done
Building dependency tree
Reading state information... Done
Calculating upgrade... Done
0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.

$ sudo apt install postgresql postgresql-contrib
$ psql --version
psql (PostgreSQL) 12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)

$ sudo service postgresql status
12/main (port 5432): down

$ sudo service postgresql start
 * Starting PostgreSQL 12 database server
$ sudo service postgresql status
12/main (port 5432): online

$ sudo service postgresql stop
 * Stopping PostgreSQL 12 database server                                                                        [ OK ]
$ sudo service postgresql status
12/main (port 5432): down

$ sudo passwd postgres
New password:
Retype new password:
passwd: password updated successfully

$ sudo apt install python3-pip
[sudo] password for username:

$ sudo pip3 install virtualenv

$ virtualenv venv

$ source venv/bin/activate

$ vi ~/.bashrc

export AIRFLOW_HOME=/mnt/c/airflow-test

$ source ~/.bashrc

echo $AIRFLOW_HOME
/mnt/c/airflow-test

$ pip3 install 'apache-airflow[postgres, slack, celery]'

$ airflow db init

$ airflow webserver -p 8081

$ airflow scheduler

$ dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart
$ dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart

$ wsl --set-default-version 2
WSL 2와의 주요 차이점에 대한 자세한 내용은 https://aka.ms/wsl2를 참조하세요

import sklearn
print("sklearn ver.", sklearn.__version__)

from sklearn.preprocessing import OneHotEncoder
enc = OneHotEncoder(handle_unknown='ignore')
X = [['Male', 1], ['Female', 3], ['Female', 2]]
enc.fit_transform(X).toarray()

from sklearn.preprocessing import OneHotEncoder
from seaborn import load_dataset

penguins = load_dataset('penguins')
ohe = OneHotEncoder()
transformed = ohe.fit_transform(penguins[['island']])
print(transformed.toarray())
print(ohe.categories_)
print(penguins['island'].unique())

print(penguins.head())

penguins[ohe.categories_[0]] = transformed.toarray()
print(penguins.head())

from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.compose import make_column_transformer
from seaborn import load_dataset
import pandas as pd

penguins = load_dataset('penguins')
sample_cols = ['island', 'sex', 'bill_length_mm', 'species']
penguins = penguins[sample_cols]

# 결측치 제거 
penguins = penguins.dropna()
print(penguins.head())
print(penguins.info())

categorical_cols = ['island', 'sex']
label_cols = ['species']

transformer = make_column_transformer(
    (OneHotEncoder(), categorical_cols),
    remainder='passthrough', 
    verbose_feature_names_out = False)

transformed = transformer.fit_transform(penguins)
transformed_df = pd.DataFrame(transformed, columns=transformer.get_feature_names_out())
print(transformed_df.head())

import pandas as pd
from seaborn import load_dataset

tips = load_dataset('tips')

# 결측치 제거 
tips = tips.dropna()
print(tips.info())

import warnings
import sklearn
import pandas as pd
import numpy as np

def get_feature_names(column_transformer):
    """Get feature names from all transformers.
    Returns
    -------
    feature_names : list of strings
        Names of the features produced by transform.
    """
    # Remove the internal helper function
    #check_is_fitted(column_transformer)
    
    # Turn loopkup into function for better handling with pipeline later
    def get_names(trans):
        # >> Original get_feature_names() method
        if trans == 'drop' or (
                hasattr(column, '__len__') and not len(column)):
            return []
        if trans == 'passthrough':
            if hasattr(column_transformer, '_df_columns'):
                if ((not isinstance(column, slice))
                        and all(isinstance(col, str) for col in column)):
                    return column
                else:
                    return column_transformer._df_columns[column]
            else:
                indices = np.arange(column_transformer._n_features)
                return ['x%d' % i for i in indices[column]]
        if not hasattr(trans, 'get_feature_names'):
        # >>> Change: Return input column names if no method avaiable
            # Turn error into a warning
            warnings.warn("Transformer %s (type %s) does not "
                                 "provide get_feature_names. "
                                 "Will return input column names if available"
                                 % (str(name), type(trans).__name__))
            # For transformers without a get_features_names method, use the input
            # names to the column transformer
            if column is None:
                return []
            else:
                return [name + "__" + f for f in column]

        return [name + "__" + f for f in trans.get_feature_names()]
    
    ### Start of processing
    feature_names = []
    
    # Allow transformers to be pipelines. Pipeline steps are named differently, so preprocessing is needed
    if type(column_transformer) == sklearn.pipeline.Pipeline:
        l_transformers = [(name, trans, None, None) for step, name, trans in column_transformer._iter()]
    else:
        # For column transformers, follow the original method
        l_transformers = list(column_transformer._iter(fitted=True))
    
    
    for name, trans, column, _ in l_transformers: 
        if type(trans) == sklearn.pipeline.Pipeline:
            # Recursive call on pipeline
            _names = get_feature_names(trans)
            # if pipeline has no transformer that returns names
            if len(_names)==0:
                _names = [name + "__" + f for f in column]
            feature_names.extend(_names)
        else:
            feature_names.extend(get_names(trans))
    
    return feature_names

from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline

categorical_cols = ['sex', 'day']
ordinal_cols = ['smoker', 'time']
numeric_cols = ['total_bill']
keep_features = [x for x in tips.columns if x not in categorical_cols + ordinal_cols + numeric_cols]

tips2 = tips[categorical_cols + ordinal_cols + numeric_cols]

transformer = ColumnTransformer(
    [('StandardScaler', StandardScaler(), numeric_cols),
     ('OneHotEncoder', OneHotEncoder(), categorical_cols),
     ('OrdinalEncoder', OrdinalEncoder(), ordinal_cols)],
    remainder='passthrough', 
    verbose_feature_names_out = False)

transformed = transformer.fit_transform(tips2)
transformed_df = pd.DataFrame(transformed, columns=get_feature_names(transformer))
tip3 = pd.concat([tips[keep_features], transformed_df], axis = 1)
tip3.info()

!pip install -U matplotlib

%matplotlib inline 

import sklearn
print(sklearn.__version__)
import matplotlib
print(matplotlib.__version__)

# 필수 라이브러리 불러오기
from sklearn.datasets import load_iris
from sklearn import tree 
import matplotlib.pyplot as plt

# 데이터 불러오기
iris = load_iris()
print(iris.data.shape, iris.target.shape)
print("feature names", iris.feature_names)
print("class names", iris.target_names)

# 모형 학습 및 plot_tree 그래프 구현
dt = tree.DecisionTreeClassifier(random_state=0)
dt.fit(iris.data, iris.target)

fig, ax = plt.subplots(figsize=(10, 6))
ax = tree.plot_tree(dt, max_depth = 2, filled=True, feature_names = iris.feature_names, class_names = iris.target_names)
plt.show()

> library(dplyr)
> library(readxl)
> raw_df = read_excel("data/NewsResult_20211213-20220313.xlsx", sheet = 1)
> raw_df2 = raw_df %>% select(일자, 언론사, 제목, URL)
> raw_df2 %>% group_by(언론사) %>% summarise(n = n())
# A tibble: 3 × 2  
언론사       n  
<chr>    <int>
1 국민일보   180
2 조선일보   115 
3 중앙일보   256

> kmib_df = raw_df2 %>% filter(언론사 == "국민일보")
> head(kmib_df, 3)
# A tibble: 3 × 4  일자     언론사   제목                                         URL                   
<chr>    <chr>    <chr>                                        <chr>               
1 20220312 국민일보 팬데믹의 비극 무너지는 가정, 스러지는 아이들 http://news.kmib.co…
2 20220312 국민일보 ‘장로’ 디딤돌인가 걸림돌인가                 http://news.kmib.co…
3 20220311 국민일보 [기고]대전은 우리가 지킨다                   http://news.kmib.co…

> kmib_df$URL[1:5]
[1] "http://news.kmib.co.kr/article/view.asp?arcid=0924235144&code=11131100"      
[2] "http://news.kmib.co.kr/article/view.asp?arcid=0924235120&code=23111111"      
[3] "http://news.kmib.co.kr/article/view.asp?arcid=0016856942&code=61221514&cp=kd"
[4] "http://news.kmib.co.kr/article/view.asp?arcid=0016853803&code=61111711&cp=kd"
[5] "http://news.kmib.co.kr/article/view.asp?arcid=0016847353&code=61131111&cp=kd"

> url = kmib_df$URL[1]
> news = read_html(url, encoding = "EUC-KR")
> news
{html_document}<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="ko">
[1] <head>\n<title>팬데믹의 비극… 무너지는 가정, 스러지는 아이들-국민일보</title>\n<meta http-equiv="Cont ...
[2] <body>\r\n<div id="wrap">\r\n\r\n<!-- header -->\r\n\r\n<script src="http://ww ...