关键字:golang、goroutine、channel、要点
时间:2018年8月

Golang编程的关键

golang编程语言相比其他语言的关键有两个,一个是goroutine,另一个是channel

关于goroutine

goroutine,相较于其他编程语言,可以理解为创建创建线程。但其他语言你需要控制线程的总数不能过多,否则CPU会大量消耗到线程切换上。但goroutine则完全不用顾虑,你可以任意调用。举个例子:

for i:=0;i<100000;i++ {
    go time.Sleep(time.Second * 300)
}

设想一下:如果你使用其他语言,创建了10万个线程,每个线程进行300秒的sleep,CPU会是什么状态?

关于channel

channel好比是阻塞队列,当队列为空时,取数据是会阻塞等待的;当队列满时,放入数据时是会阻塞等待的。

c := make(chan int, 10)
for i:=0;i<20;i++ {
    go func() {
        c <- i
        fmt.Printf("%d\n", i) // 通道c中放满10个数据后,则进入阻塞状态,此处不再打印。
    }()
}
time.Sleep(time.Second * 10)
for j:=0;j<10;j++ {
    <- c // 从通道c中取出数据后,则继续打印i。
 
}
c := make(chan int, 10)
for i:=0;i<10;i++ {
    go func() {
        n <- c
        fmt.Printf("%d\n", i) // 通道c中有数据后,才会执行此处的打印。
    }()
}
time.Sleep(time.Second * 10)
for j:=0;j<10;j++ {
    c <- j // 向通道c中放入数据,则开始打印i。
}

其他

像sync.WaitGroup等都是通过channel实现。

关键字:debian、elk、安装、elasticsearch、kibana、logstash、nginx
时间:2018年8月

0. 环境

Debian GNU/Linux 9

1.安装Java运行环境

root@debian:~# apt-get install openjdk-8-jdk

2. 安装配置ElasticSearch

root@debian:~# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb
root@debian:~# dpkg -i elasticsearch-6.3.2.deb
...
root@debian:~# vim /etc/elasticsearch/elasticsearch.yml
...
#network.host:192.168.0.1
network.host:172.17.1.100
...
:wq
root@debian:~# systemctl start elasticsearch
root@debian:~# ss -ant|grep 9200
LISTEN     0      128    ::ffff:172.17.1.100:9200                    :::*

root@debian:~# curl http://172.17.1.100:9200/_search?pretty
{
  "took" : 14,
  "timed_out" : false,
  "_shards" : {
    "total" : 0,
    "successful" : 0,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 0,
    "max_score" : 0.0,
    "hits" : [ ]
  }
}

3. 安装Logstash

root@debian:~# wget https://artifacts.elastic.co/downloads/logstash/logstash-6.3.2.deb
root@debian:~# dpkg -i logstash-6.3.2.deb
root@debian:~# systemctl start logstash

4. 安装配置Kibana

root@debian:~# wget https://artifacts.elastic.co/downloads/kibana/kibana-6.3.2-amd64.deb
root@debian:~# dpkg -i kibana-6.3.2-amd64.deb
root@debian:~# vim /etc/kibana/kibana.yml
...
#server.host: "localhost"
server.host: "172.17.1.100"
...
#elasticsearch.url: "http://localhost:9200"
elasticsearch.url: "http://172.17.1.100:9200"
...
:wq
root@debian:~# systemctl start kibana
root@debian:~# ss -ant|grep 5601
root@debian:~# ss -ant|grep 5601
root@debian:~# ss -ant|grep 5601
LISTEN     0      128    172.17.1.100:5601                     *:*

注:启动kibana的时间有点长,请耐心等待。

5. Logstash收集Nginx的access日志

root@debian:~# apt-get intall nginx
root@debian:~# vim /etc/logrotate.d/nginx
...
	create 0644 www-data adm
...
:wq
root@debian:~# curl http://localhost/
root@debian:~# chmod 644 /var/log/nginx/*
root@debian:~# vim /etc/logstash/conf.d/nginx_access.conf
input {
	file {
		path => ["/var/log/nginx/access.log"]
		start_position => "beginning"
	}
}

filter {
	grok {
		match => {
			"message" => "%{HTTPD_COMBINEDLOG}"
		}
	}
	urldecode {
		all_fields => true
	}
}

output {
	elasticsearch {
		hosts => ["172.17.1.100:9200"]
		index => "logstash-nginx-access-%{+YYYY.MM.dd}"
	}
	stdout {
		codec => rubydebug
	}
}
:wq
root@debian:~# systemctl restart logstash

注:/var/log/nginx/access.log默认权限为600,logstash无法访问,需要修改为644。

6. Kibana查看nginx日志

6.1. 配置kibana

1、浏览器打开http://172.17.1.100:5601/
2、点击Managment菜单,显示Create Index Patterns页面
3、Step 1 of 2: Define index pattern页面,填写”logstash-nginx-access-*”(引号不填) -> 点击”Next Step”
4、Step 2 of 2: Configure settings页面,下拉选择”@timestamp”,点击”Create index pattern”

制造Nginx日志

root@debian:~# curl http://172.17.1.100/
root@debian:~# curl http://172.17.1.100/
root@debian:~# curl http://172.17.1.100/

6.2. 搜索日志

1、浏览器打开http://172.17.1.100:5601/
2、点击Discover菜单,点击放大镜按钮。

7. 补充

如需开机启动,请执行如下脚本。

root@debian:~# systemctl enable elasticsearch
root@debian:~# systemctl enable logstash
root@debian:~# systemctl enable kibana

关键字:mac、iso、u盘、启动盘
时间:2018年8月

iso格式转换

不确定哪些u盘需要进行此操作。如果遇到BIOS启动选项无法正常识别,请进行该操作。

MacBook-Air:Downloads user1$ hdiutil convert -format UDRW -o debian.iso debian-9.5.0-amd64-netinst.iso 
Reading Driver Descriptor Map (DDM : 0)…
Reading Debian 9.5.0 amd64 n             (Apple_ISO : 1)…
Reading Apple (Apple_partition_map : 2)…
Reading Debian 9.5.0 amd64 n             (Apple_ISO : 3)…
Reading EFI (Apple_HFS : 4)…
.
Reading Debian 9.5.0 amd64 n             (Apple_ISO : 5)…
...................................................................................................................................................
Elapsed Time:  2.985s
Speed: 97.5Mbytes/sec
Savings: 0.0%
created: /Users/user1/Downloads/debian.iso.dmg
MacBook-Air:Downloads user1$ mv debian.iso.dmg debian.iso

写入u盘

请仔细核对,不要弄错IDENTIFIER,丢失数据。

MacBook-Air:Downloads user1$ diskutil list
/dev/disk0 (internal, physical):
   #:                       TYPE NAME                    SIZE       IDENTIFIER
   0:      GUID_partition_scheme                        *121.3 GB   disk0
   1:                        EFI EFI                     209.7 MB   disk0s1
   2:                 Apple_APFS Container disk1         121.1 GB   disk0s2

/dev/disk1 (synthesized):
   #:                       TYPE NAME                    SIZE       IDENTIFIER
   0:      APFS Container Scheme -                      +121.1 GB   disk1
                                 Physical Store disk0s2
   1:                APFS Volume Macintosh HD            105.4 GB   disk1s1
   2:                APFS Volume Preboot                 20.0 MB    disk1s2
   3:                APFS Volume Recovery                516.2 MB   disk1s3
   4:                APFS Volume VM                      2.1 GB     disk1s4

/dev/disk2 (internal, physical):
   #:                       TYPE NAME                    SIZE       IDENTIFIER
   0:     FDisk_partition_scheme                        *32.4 GB    disk2
   1:               Windows_NTFS SD                      32.4 GB    disk2s1

/dev/disk3 (external, physical):
   #:                       TYPE NAME                    SIZE       IDENTIFIER
   0:     FDisk_partition_scheme                        *8.0 GB     disk3
   1:                 DOS_FAT_32 NO NAME                 8.0 GB     disk3s1

MacBook-Air:Downloads user1$ diskutil unmountDisk /dev/disk3
Unmount of all volumes on disk3 was successful
MacBook-Air:Downloads user1$ sudo dd if=./debian.iso of=/dev/rdisk3 bs=1m
Password:
291+0 records in
291+0 records out
305135616 bytes transferred in 61.626893 secs (4951339 bytes/sec)
MacBook-Air:Downloads user1$ diskutil eject /dev/disk3
Disk /dev/disk3 ejected

注:dd时,使用/dev/rdisk*比/dev/disk*要快不少。

关键字:fastdfs、安装、配置
时间:2018年8月

前言

1、截止编写时,FastDFS未提供官方DEB安装包,所以需采用编译安装的方式进行安装。
2、FastDFS依赖libfastcommon库,apt包管理中无法找到此库,因此只能选择编译安装。
3、happyfish100就是FastDFS的作者,不是fork。

编译安装libfastcommon

root@debian:~# wget https://github.com/happyfish100/libfastcommon/archive/master.zip
root@debian:~# unzip master.zip
root@debian:~# cd libfastcommon-master
root@debian:~/libfastcommon-master# ./make.sh
root@debian:~/libfastcommon-master# ./make.sh install

编译安装FastDFS

root@debian:~# wget https://github.com/happyfish100/fastdfs/archive/V5.11.tar.gz
root@debian:~# tar -zxvf V5.11.tar.gz
root@debian:~# cd fastdfs-5.11/
root@debian:~/fastdfs-5.11# ./make.sh
root@debian:~/fastdfs-5.11# ./make.sh install

配置启动tracker

root@debian:~# cd /etc/fdfs/
root@debian:/etc/fdfs# cp tracker.conf.sample tracker.conf
root@debian:/etc/fdfs# vim tracker.conf
...
#base_path=/home/yuqing/fasdfs
base_path=/srv/fastdfs/tracker
...
:wq
root@debian:~# cd /srv
root@debian:/srv# mkdir fastdfs
root@debian:/srv# mkdir fastdfs/tracker
root@debian:/# /etc/init.d/fdfs_trackerd start

配置启动storage

root@debian:~# cd /etc/fdfs/
root@debian:/etc/fdfs# cp storage.conf.sample storage.conf
root@debian:/etc/fdfs# vim /etc/storage.conf
...
#base_path=/home/yuqing/fasdfs
base_path=/srv/fastdfs/storage
...
#store_path0=/home/yuping/fastdfs
#store_path1=/home/yuping/fastdfs2
store_path0=/srv/fastdfs/storage
...
tracker_server=192.168.1.100:22122
...
:wq
root@debian:/etc/fdfs# cd /srv
root@debian:/srv# mkdir fastdfs/storage
root@debian:/srv# /etc/init.d/fdfs_storaged start

查看FastDFS状态

root@debian:~# fdfs_monitor /etc/fdfs/storage.conf|more
[2018-08-04 13:49:38] DEBUG - base_path=/srv/fastdfs/storage, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0

server_count=1, server_index=0

tracker server is 172.17.1.100:22122

group count: 1

Group 1:
group name = group1
disk total space = 28030 MB
disk free space = 24745 MB
trunk free space = 0 MB
storage server count = 1
active server count = 1
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0

	Storage 1:
		id = 172.17.1.100
		ip_addr = 172.17.1.100  ACTIVE
		http domain = 
		version = 5.11
		join time = 2018-08-04 13:36:45
		up time = 2018-08-04 13:39:12
		total storage = 28030 MB
		free storage = 24745 MB
		upload priority = 10
		store_path_count = 1
		subdir_count_per_path = 256
		storage_port = 23000
		storage_http_port = 8888
		current_write_path = 0
		source storage id = 
		if_trunk_server = 0
		connection.alloc_count = 256
		connection.current_count = 0
		connection.max_count = 0
		total_upload_count = 0
		success_upload_count = 0
		total_append_count = 0
		success_append_count = 0
		total_modify_count = 0
		success_modify_count = 0
		total_truncate_count = 0
		success_truncate_count = 0
		total_set_meta_count = 0
		success_set_meta_count = 0
		total_delete_count = 0
		success_delete_count = 0
		total_download_count = 0
		success_download_count = 0
		total_get_meta_count = 0
		success_get_meta_count = 0
		total_create_link_count = 0
		success_create_link_count = 0
		total_delete_link_count = 0
		success_delete_link_count = 0
		total_upload_bytes = 0
		success_upload_bytes = 0
		total_append_bytes = 0
		success_append_bytes = 0
		total_modify_bytes = 0
		success_modify_bytes = 0
		stotal_download_bytes = 0
		success_download_bytes = 0
		total_sync_in_bytes = 0
		success_sync_in_bytes = 0
		total_sync_out_bytes = 0
		success_sync_out_bytes = 0
		total_file_open_count = 0
		success_file_open_count = 0
		total_file_read_count = 0
		success_file_read_count = 0
		total_file_write_count = 0
		success_file_write_count = 0
		last_heart_beat_time = 2018-08-04 13:49:11
		last_source_update = 1970-01-01 08:00:00
		last_sync_update = 1970-01-01 08:00:00
		last_synced_timestamp = 1970-01-01 08:00:00