bug fixes (#18)

Author: hilldani

.gitignore

@@ -2,3 +2,4 @@ build/*
 dist/*
 pmu-checker/pmu-checker
 src/libtsc.so
+__pycache__

Makefile

@@ -4,13 +4,13 @@ VERSION_FILE := _version.txt
 VERSION_BASE := $(COMMIT_DATE)_$(COMMIT_ID)
 VERSION_NUMBER := $(shell cat ${VERSION_FILE})
 VERSION_PUBLIC := $(VERSION_NUMBER)
-PACKAGE_EXTERNAL := perfspect_$(VERSION_NUMBER).tgz
+PACKAGE_EXTERNAL := perfspect.tgz
 BINARY_FINAL := perfspect
 BINARY_COLLECT := perf-collect
 BINARY_POSTPROCESS := perf-postprocess
-default: all
+default: dist
 
-.PHONY: all test default dist clean format format_check security_scan flakes source_check checkmake dist/version_file dist/$(SOURCE_PACKAGE)
+.PHONY: test default dist format format_check style_error_check check dist/version_file dist/$(SOURCE_PACKAGE)
 
 clean_dir:
 	rm -rf build/*
@@ -34,22 +34,9 @@ build-public/collect:
 	mkdir -p $(TMPDIR)/src
 	mkdir -p $(TMPDIR)/events
 	cp src/* $(TMPDIR)/src && cp events/* $(TMPDIR)/events && cp *.py $(TMPDIR)
-	sed -i 's/PerfSpect_DEV_VERSION/$(VERSION_PUBLIC)/g' $(TMPDIR)/src/perf_helpers.py 
-	cd $(TMPDIR) && pyinstaller -F perf-collect.py -n $(BINARY_COLLECT) \
-					--add-data "./src/libtsc.so:." \
-					--add-data "./events/bdx.txt:." \
-					--add-data "./events/skx.txt:." \
-					--add-data "./events/clx.txt:." \
-					--add-data "./events/icx.txt:." \
-					--add-data "./events/spr.txt:." \
-					--add-data "./events/icx_aws.txt:." \
-					--add-data "./events/spr_aws.txt:." \
-					--add-data "./events/clx_aws.txt:." \
-					--add-data "./events/skx_aws.txt:." \
-					--add-binary "../build/pmu-checker:." \
-					--runtime-tmpdir . \
-					--exclude-module readline
-	
+	sed -i 's/PerfSpect_DEV_VERSION/$(VERSION_PUBLIC)/g' $(TMPDIR)/src/perf_helpers.py
+	cp perf-collect.spec $(TMPDIR)
+	cd $(TMPDIR) && pyinstaller perf-collect.spec
 	cp $(TMPDIR)/dist/$(BINARY_COLLECT) build/
 	rm -rf $(TMPDIR)
 
@@ -87,18 +74,16 @@ test:
 	cd dist && tar -xvf perfspect.tgz && cp -r $(BINARY_FINAL) ../test/.
 	cd test && pytest
 
-format:
-	black src
-	black *.py
-
 format_check:
-	black --check src
-	black --check perf-collect.py perf-postprocess.py
+	black --check *.py src
+
+format:
+	black *.py src
 
-error_check: # ignore false positives
-	flake8 --ignore=E501,W503,F403,F405,E741 src
-	flake8 --ignore=E203,E501,E722,W503,F403,F405 *.py --exclude simpleeval.py,perfmon.py,average.py
+style_error_check:
+	# ignore long lines and conflicts with black, i.e., black wins
+	flake8 *.py src --ignore=E501,W503,E203
 
-source_check: security_scan format_check error_check
+check: format_check style_error_check
 
-dist: source_check dist/$(PACKAGE_EXTERNAL) 
+dist: check dist/$(PACKAGE_EXTERNAL) 

README.md

@@ -25,7 +25,7 @@ Requires recent python and golang.
 
 ```
 pip3 install -r requirements.txt
-make dist
+make
 ```
 
 On successful build, binaries will be created in "dist" folder

perf-collect.py

@@ -519,9 +519,11 @@ def is_safe_file(fname, substr):
         False,
     )
 
+    # reset nmi_watchdog to what it was before running perfspect
     if (int(nmi_watchdog) != 0) and supervisor:
         f_nmi = open("/proc/sys/kernel/nmi_watchdog", "w")
         f_nmi.write(nmi_watchdog)
+        f_nmi.close()
 
     if (args.muxinterval > 0) and supervisor:
         perf_helpers.set_perf_event_mux_interval(True, 1, mux_intervals)

perf-collect.spec

@@ -0,0 +1,49 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+block_cipher = None
+
+
+a = Analysis(
+    ['perf-collect.py'],
+    pathex=[],
+    binaries=[('../build/pmu-checker', '.')],
+    datas=[('./src/libtsc.so', '.'), ('./events/bdx.txt', '.'), ('./events/skx.txt', '.'), ('./events/clx.txt', '.'), ('./events/icx.txt', '.'), ('./events/spr.txt', '.'), ('./events/icx_aws.txt', '.'), ('./events/spr_aws.txt', '.'), ('./events/clx_aws.txt', '.'), ('./events/skx_aws.txt', '.')],
+    hiddenimports=[],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=['readline'],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+
+# exclude libtinfo shared library from distributed binaries due to warning observed on Ubuntu 16.04:
+#    "/bin/bash: ./_MEIuU3XMv/libtinfo.so.5: no version information available (required by /bin/bash)"
+a.binaries = [bin for bin in a.binaries if not bin[0].startswith('libtinfo')]
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    [],
+    name='perf-collect',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir='.',
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)

perf-postprocess.py

@@ -763,8 +763,6 @@ def write_perf_tmp_output(use_epoch):
                     epoch = int(words[-1])
                 except ValueError:
                     exit("Conversion error parsing timestamp")
-                except:
-                    exit("Unkown error parsing timestamp")
                 break
     # TO:DO remove "not_counted" and "not_supported" events from dat_file
 
@@ -1433,7 +1431,7 @@ def is_safe_path(base_dir, path, follow_symlinks=True):
     if (args.outfile).endswith("xlsx"):
         try:
             import xlsxwriter
-        except:
+        except ImportError:
             raise SystemExit(
                 "xlsxwriter not found to generate excel output. Install xlsxwriter or use .csv"
             )

pmu-checker/msr/msr.go

@@ -52,7 +52,7 @@ func (dpt retMSR) read(msr int64) (uint64, error) {
 	rc, err := syscall.Pread(dpt.fd, buf, msr)
 	if err != nil {
 		log.Fatal(err)
-		panic(err)
+		return 0, err
 	}
 
 	if rc != 8 {

src/perf_helpers.py

@@ -14,8 +14,9 @@
 import collections
 import psutil
 import subprocess  # nosec
+import logging
 from time import strptime
-from ctypes import *  # flake8: noqa
+from ctypes import cdll, CDLL
 from datetime import datetime
 from dateutil import tz
 
@@ -166,7 +167,7 @@ def get_version():
     try:
         fo = open("/proc/version", "r")
     except EnvironmentError as e:
-        warnings.warn(str(e), UserWarning)
+        logging.warn(str(e), UserWarning)
     else:
         version = fo.read()
         version = version.split("#")[0]
@@ -180,7 +181,7 @@ def get_cpuinfo():
     try:
         fo = open("/proc/cpuinfo", "r")
     except EnvironmentError as e:
-        warnings.warn(str(e), UserWarning)
+        logging.warn(str(e), UserWarning)
     else:
         for line in fo:
             try: