From 2c0716d2e5a5b17f3f6583e57c59144088172044 Mon Sep 17 00:00:00 2001
From: Saara Saaninkoski <saara.saaninkoski@tuni.fi>
Date: Wed, 29 Nov 2023 11:25:11 +0000
Subject: [PATCH] Set up Jenkins integration
---
.coveragerc | 2 +
.gitignore | 4 +-
.pylintrc | 2 +
Jenkinsfile | 159 +++++++++++++++++++++++++++++
MANIFEST.in | 21 ++--
VERSION | 1 +
kvalikirstu2/utils.py | 3 +-
requirements.txt | 19 ++--
requirements_test.txt | 20 +---
setup.py | 11 +-
sonar-project.properties | 9 ++
tests/conftest.py | 5 +-
tests/test_argument_parser.py | 1 +
tests/test_backup_select_window.py | 5 +
tests/test_converter.py | 3 +
tests/test_csv_generator.py | 2 +-
tests/test_gui.py | 6 ++
tests/test_gui_model.py | 1 +
tests/test_gui_utils.py | 6 ++
tests/test_serialization.py | 6 +-
tests/test_tmp_edit_window.py | 6 ++
tests/test_warning_handler.py | 15 ++-
tox.ini | 29 +++++-
23 files changed, 277 insertions(+), 59 deletions(-)
create mode 100644 .coveragerc
create mode 100644 .pylintrc
create mode 100644 Jenkinsfile
create mode 100644 VERSION
create mode 100644 sonar-project.properties
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..df29a12
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+source=kvalikirstu2
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 0371e0f..73c1f4b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,11 +25,11 @@ docs/_static
docs/_templates
.hgignore
.babelrc
-.coveragerc
.hg/
.hgtags
.idea/
-.pylintrc
.tox
dist/
kvalikirstu2.egg-info/
+build/*
+tests/test files/*
\ No newline at end of file
diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 0000000..a63f80b
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,2 @@
+[FORMAT]
+max-line-length=120
\ No newline at end of file
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..de9ead0
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,159 @@
+#!/usr/bin/env groovy
+
+node_name = 'default'
+email_recipient = ''
+pylint_targets = 'kvalikirstu2'
+
+def sendmail(build_result) {
+ stage('Send mail') {
+ mail(to: email_recipient,
+ subject: "Job '${env.JOB_NAME}' (${env.BUILD_NUMBER}) result is ${build_result}",
+ body: "See info at ${env.BUILD_URL}")
+ }
+}
+
+// Discard old builds
+properties([buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '30'))])
+
+// node reserves executor and workspace
+node(node_name) {
+ // Prepare
+ // -------
+ def toxEnvName = '.env-tox'
+ def pylintEnvName = '.env-pylint'
+ def sqScannerHome = tool 'SonarQube Scanner'
+ def pylint_report_path = 'pylint_report.txt'
+ def coverage_xml_path = 'coverage.xml'
+ def sonar_properties_path = 'sonar-project.properties'
+
+ // prepare workspace
+ def myworkspace = ''
+ // Get current node, to force concurrent steps to be run on the same node.
+ def current_node = "${env.NODE_NAME}"
+
+ // Parallelism causes stages to be run in different workspaces.
+ // Before submitting to SonarQube we need to make sure pylint_report
+ // coverage.xml and sonar-project.properties files are in-place.
+ // tasks shall be run in parallel
+ def tasks_1 = [:]
+ def tasks_2 = [:]
+
+ myworkspace = "${WORKSPACE}"
+ echo "My workspace is ${myworkspace}"
+ deleteDir()
+
+ // Get recipient from git commit author
+ def scmVars = checkout scm
+ email_recipient = sh (
+ script: "git show -s --pretty=%ae",
+ returnStdout: true
+ )
+ echo "Build result will be sent to ${email_recipient}"
+
+ // Assign parallel tasks
+ tasks_1['Prepare Tox, Run With Coverage & Publish Report'] = {
+ node(current_node) {
+ dir(myworkspace) {
+ stage('Prepare Tox Venv') {
+ if (!fileExists(toxEnvName)) {
+ echo 'Build Python Virtualenv for testing...'
+ sh """
+ python-latest -m venv ${toxEnvName}
+ . ./${toxEnvName}/bin/activate
+ pip install --upgrade pip
+ pip install tox
+ """
+ }
+ }
+ stage('Run Test Suite & Gather Coverage') {
+ sh """
+ . ./${toxEnvName}/bin/activate
+ tox -e jenkins-with-coverage
+ """
+ }
+ stage('Publish Cobertura Report') {
+ cobertura(coberturaReportFile: "${coverage_xml_path}",
+ zoomCoverageChart: false)
+ }
+ }
+ }
+ }
+ tasks_1['Prepare Pylint, Run Analysis, Archive & Publish report'] = {
+ node(current_node) {
+ dir(myworkspace) {
+ stage('Prepare Pylint Venv') {
+ if (!fileExists(pylintEnvName)) {
+ echo 'Build Python Virtualenv for linting...'
+ sh """
+ python-latest -m venv ${pylintEnvName}
+ . ./${pylintEnvName}/bin/activate
+ pip install --upgrade pip
+ pip install -r ./requirements.txt
+ pip install .
+ pip install pylint
+ """
+ }
+ }
+ stage('Run PyLint') {
+ echo 'Run pylint'
+ sh """
+ . ./${pylintEnvName}/bin/activate
+ pylint -f parseable ${pylint_targets} | tee ${pylint_report_path}
+ """
+ }
+ stage('Archive PyLint Report') {
+ archiveArtifacts artifacts: pylint_report_path
+ }
+ stage('Publish PyLint Report') {
+ recordIssues tool: pyLint(pattern: pylint_report_path)
+ }
+ }
+ }
+ }
+ tasks_2['Run Tests py311'] = {
+ node(current_node) {
+ dir(myworkspace) {
+ stage('Run Tests') {
+ sh """
+ . ./${toxEnvName}/bin/activate
+ tox -e jenkins
+ """
+ }
+ }
+ }
+ }
+ tasks_2['Initiate SonarQube Analysis'] = {
+ node(current_node) {
+ dir(myworkspace) {
+ stage('Prepare sonar-project.properties') {
+ sh "echo sonar.projectVersion = \$(cat VERSION) >> ${sonar_properties_path}"
+ }
+ stage('Initiate SonarQube analysis') {
+ withSonarQubeEnv() {
+ sh "${sqScannerHome}bin/sonar-scanner"
+ }
+ }
+ }
+ }
+ }
+ try {
+ // run parallel tasks
+ parallel tasks_1
+ parallel tasks_2
+ } catch (err) {
+ currentBuild.result = 'FAILURE'
+ sendmail('FAILURE')
+ }
+}
+// Wait for sonar quality gate
+stage("Quality Gate") {
+ timeout(time: 10, unit: 'MINUTES') { // Just in case something goes wrong, pipeline will be killed after a timeout
+ def qg = waitForQualityGate() // Reuse taskId previously collected by withSonarQubeEnv
+ if (qg.status != 'OK') {
+ echo "Pipeline unstable due to quality gate failure: ${qg.status}"
+ currentBuild.result = 'UNSTABLE'
+ sendmail('UNSTABLE')
+ }
+ }
+}
+
diff --git a/MANIFEST.in b/MANIFEST.in
index 16cb432..baf28d9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,14 +1,11 @@
-include README.txt
-recursive-include tests *
-recursive-include kvalikirstu2 *
-recursive-include docs *
-recursive-include "kvalikirstu2/locale" *
-include scripts/*
-include scripts/docs/*
-include *.include
-include *.ini
+include VERSION
+include *.rst
include *.txt
+include tox.ini
+include .pylintrc
+include .coveragerc
+include tests/*
-global-exclude *.py[co]
-global-exclude __pycache__
-prune kvalikirstu2/.vscode
\ No newline at end of file
+recursive-include "tests/test files src" *
+recursive-include tests *
+recursive-include "kvalikirstu2/locale" *
\ No newline at end of file
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..345f8cc
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+0.1.10
\ No newline at end of file
diff --git a/kvalikirstu2/utils.py b/kvalikirstu2/utils.py
index e82c4e3..14c5376 100644
--- a/kvalikirstu2/utils.py
+++ b/kvalikirstu2/utils.py
@@ -153,11 +153,10 @@ def json_serialize(obj, path: str):
"""
logger.debug('Saving %s to path %s', obj, path)
- json_string = jsonpickle.encode(obj)
+ json_string = jsonpickle.encode(obj, make_refs=False)
obj = json.loads(json_string)
output = json.dumps(obj, indent=4, ensure_ascii=False, sort_keys=True).encode('utf8')
create_folder_if_not_exists(os.path.dirname(path))
-
with open(path, 'wb') as file_stream:
file_stream.write(output)
diff --git a/requirements.txt b/requirements.txt
index 7fc8b60..44b7ef1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,13 +1,14 @@
-Babel>=2.8.0
-chardet==3.0.4
+Babel==2.13.1
+chardet==5.2
ConfigArgParse==0.14.0
defusedxml==0.5.0
-Jinja2==2.10
-jsonpickle==1.1
-MarkupSafe==1.1.1
-natsort==6.2.0
+Jinja2==3.0.3
+jsonpickle==3.0.2
+MarkupSafe==2.1.3
+natsort==8.4.0
odfpy==1.4.0
-Pillow>=6.0.0
-six>=1.14.0
-wxPython==4.1.1;platform_system=="Windows"
+Pillow==10.1.0
+setuptools==58
+six==1.16.0
+wxPython==4.2.1;platform_system=="Windows"
pypubsub===4.0.3
\ No newline at end of file
diff --git a/requirements_test.txt b/requirements_test.txt
index 9250700..0fcad43 100644
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -1,17 +1,3 @@
-Babel==2.6.0
-chardet==3.0.4
-ConfigArgParse==0.14.0
-defusedxml==0.5.0
-Jinja2==2.10
-jsonpickle==1.1
-MarkupSafe==1.1.1
-natsort==6.2.0
-odfpy==1.4.0
-Pillow>=6.0.0
-six>=1.14.0
-wxPython==4.1.1;platform_system=="Windows"
-https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-16.04/wxPython-4.1.1-cp38-cp38-linux_x86_64.whl; sys_platform == "linux"
-pytest<=4.0
-pytest-cov<=2.8
-pyvirtualdisplay==0.2.1
-pypubsub===4.0.3
\ No newline at end of file
+pytest==7.4.3
+pytest-cov==4.1.0
+pyvirtualdisplay==3.0
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 6af4de7..22996b6 100644
--- a/setup.py
+++ b/setup.py
@@ -2,12 +2,16 @@
# Author(s): Saara Saaninkoski and Jukka Lipsanen
# Copyright 2019 Finnish Social Science Data Archive FSD / University of Tampere
# Licensed under the EUPL. See LICENCE.txt for full license.
-
+import os
from setuptools import setup, find_packages
+this_dir = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(this_dir, 'VERSION'), 'r') as fileobj:
+ version = fileobj.readline().strip()
+
setup(
name='kvalikirstu2',
- version='1.0.10',
+ version=version,
url='',
description='A tool for analyzing qualitative studies',
author='Jukka Lipsanen and Saara Saaninkoski',
@@ -22,9 +26,6 @@ setup(
'pypubsub',
'natsort'
],
- tests_require=['pytest'],
- setup_requires=['Babel'],
- include_package_data=True,
entry_points={
'console_scripts': [
'python-kvalikirstu2-gui = kvalikirstu2.gui:main',
diff --git a/sonar-project.properties b/sonar-project.properties
new file mode 100644
index 0000000..9ee5b18
--- /dev/null
+++ b/sonar-project.properties
@@ -0,0 +1,9 @@
+# unique project identifier (required) - The project's unique key. Allowed characters are: letters,
+# numbers, -, _, . and :, with at least one non-digit.
+sonar.projectKey=kvalikirstu2
+# path to source directories (required) - Comma-separated paths to directories containing main source files.
+sonar.sources=kvalikirstu2
+# Path to coverage (Cobertura XML) report
+sonar.python.coverage.reportPaths=coverage.xml
+# Path to pylint report
+sonar.python.pylint.reportPath=pylint_report.txt
diff --git a/tests/conftest.py b/tests/conftest.py
index 4922255..4afba12 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -3,6 +3,8 @@ import os
import pytest
import logging
+from kvalikirstu2 import argument_parser
+
TEST_FILES_SRC = os.path.join(os.path.dirname(__file__), "test files src")
TEST_FILES_DST = os.path.join(os.path.dirname(__file__), "test files")
VARS = {'display': None}
@@ -20,10 +22,11 @@ def setup(request):
request.addfinalizer(finalize)
setup_rootlogger()
- if os.name == "posix" and "DISPLAY" not in os.environ:
+ if os.name == "posix" and (all(not os.getenv(x) for x in ["DISPLAY", "NO_WX"])):
import pyvirtualdisplay
VARS['display'] = pyvirtualdisplay.Display()
VARS['display'].start()
+ argument_parser.Settings().set_up_cli_settings([])
def rootlogger_error(*args, **kwargs):
diff --git a/tests/test_argument_parser.py b/tests/test_argument_parser.py
index 1164368..16a6223 100644
--- a/tests/test_argument_parser.py
+++ b/tests/test_argument_parser.py
@@ -82,6 +82,7 @@ class ArgumentParserTest(unittest.TestCase, test_writer.CitReqMixin, test_kvali_
self.assertTrue(mock.called)
+ @unittest.skipIf(os.getenv('NO_WX'), 'Skipping tests that require wxPython')
def test_handler_for_gui(self):
settings = argument_parser.Settings()
settings.set_up_gui_settings([])
diff --git a/tests/test_backup_select_window.py b/tests/test_backup_select_window.py
index 5b1a7d9..5515c7c 100644
--- a/tests/test_backup_select_window.py
+++ b/tests/test_backup_select_window.py
@@ -1,4 +1,9 @@
+# Skip this module if NO_WX in env
+import pytest
import os
+if os.getenv('NO_WX'):
+ pytest.skip('Skipping module because wxPython is not available', allow_module_level=True)
+
import unittest
import wx
from kvalikirstu2 import argument_parser
diff --git a/tests/test_converter.py b/tests/test_converter.py
index b9e53d8..1d8f92e 100644
--- a/tests/test_converter.py
+++ b/tests/test_converter.py
@@ -119,6 +119,7 @@ class TestConvertEncoding(unittest.TestCase):
self.subfolder_conversion_test("qwertyuiopasdfghjklzxcvbnmåäö", 'cp1252', 10)
self.subfolder_conversion_test("qwertyuiopasdfghjklzxcvbnmåäö", 'utf-8', 10)
+ @unittest.skipIf(os.getenv('NO_ODT'), 'Skipping tests that require LibreOffice')
def test_libreoffice_conversion_to_odt(self):
docx_path = os.path.join(self.temp_path, "data", "test_data1.docx")
data_path = os.path.join(self.temp_path, "data")
@@ -133,6 +134,7 @@ class TestConvertEncoding(unittest.TestCase):
self.assertFalse(os.path.exists(docx_path))
self.assertEqual(expected_content, content)
+ @unittest.skipIf(os.getenv('NO_ODT'), 'Skipping tests that require LibreOffice')
def test_libreoffice_conversion_to_txt(self):
docx_path = os.path.join(self.temp_path, "data", "test_data1.docx")
data_path = os.path.join(self.temp_path, "data")
@@ -148,6 +150,7 @@ class TestConvertEncoding(unittest.TestCase):
self.assertFalse(os.path.exists(docx_path))
self.assertEqual(lines1, lines2)
+ @unittest.skipIf(os.getenv('NO_ODT'), 'Skipping tests that require LibreOffice')
def test_timeout_low_throws(self):
data_path = os.path.join(self.temp_path, "data")
with self.assertRaises(subprocess.TimeoutExpired):
diff --git a/tests/test_csv_generator.py b/tests/test_csv_generator.py
index 1fb5c07..f27f8bb 100644
--- a/tests/test_csv_generator.py
+++ b/tests/test_csv_generator.py
@@ -24,7 +24,7 @@ class TestCSVGenerator(unittest.TestCase):
self.interface = DummyInterface()
def test_generator(self):
- reader = study_reader.StudyReader('FSDxxxx', True, data_file=get_testpath(), study_path=get_testdir())
+ reader = study_reader.StudyReader('FSDxxxx', 'Title', data_file=get_testpath(), study_path=get_testdir())
header_info = header_scanner.get_header_info_for_study(get_testpath())
study = reader.get_study()
csv_path = os.path.join(get_testdir(), 'daFxxxx.csv')
diff --git a/tests/test_gui.py b/tests/test_gui.py
index bb0d996..69abf12 100644
--- a/tests/test_gui.py
+++ b/tests/test_gui.py
@@ -1,5 +1,11 @@
""" Tests for gui classes.
"""
+# Skip this module if NO_WX in env
+import pytest
+import os
+if os.getenv('NO_WX'):
+ pytest.skip('Skipping module because wxPython is not available', allow_module_level=True)
+
import os
import unittest
import wx
diff --git a/tests/test_gui_model.py b/tests/test_gui_model.py
index bf76254..032409a 100644
--- a/tests/test_gui_model.py
+++ b/tests/test_gui_model.py
@@ -68,6 +68,7 @@ class TestGUIModel(unittest.TestCase):
self.model.save_changes_to_daf()
self.assertFalse(self.model.check_unsaved_changes())
+ @unittest.skipIf(os.getenv('NO_ODT'), 'Skipping tests that require Libreoffice')
def test_conversion(self):
self.model.convert_to_odt()
test_data_path = os.path.join(self.temp_path, "data", "to_convert.odt")
diff --git a/tests/test_gui_utils.py b/tests/test_gui_utils.py
index bc69af5..a9bbcb5 100644
--- a/tests/test_gui_utils.py
+++ b/tests/test_gui_utils.py
@@ -1,3 +1,9 @@
+# Skip this module if NO_WX in env
+import pytest
+import os
+if os.getenv('NO_WX'):
+ pytest.skip('Skipping module because wxPython is not available', allow_module_level=True)
+
import unittest
import wx
from kvalikirstu2 import argument_parser, gui_utils
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index 8e97c2b..da69036 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -11,7 +11,7 @@ from kvalikirstu2 import argument_parser
@pytest.fixture(name='testdir')
def get_testdir():
- return os.path.join(os.path.dirname(__file__), "test files", "valid")
+ return os.path.join(os.path.dirname(__file__), "test files")
@pytest.fixture(name='settings')
@@ -34,7 +34,7 @@ class TestSerialization(unittest.TestCase):
@pytest.fixture(autouse=True)
def init_reader_valid(self, testdir, test_id, store_args):
store_args
- self.reader = study_reader.StudyReader(testdir, test_id, "", study_path=testdir)
+ self.reader = study_reader.StudyReader(study_id=test_id, study_path=testdir, study_title="", data_folder_name="data")
"""For testing the serialization in the utils module"""
def test_study_reader(self):
@@ -47,7 +47,7 @@ class TestSerialization(unittest.TestCase):
loaded_study.write_to_file()
# Load a copy of the study from file
- copy = study.Study(self.reader.study_id, self.reader.study_path, "")
+ copy = study.Study(study_id=self.reader.study_id, study_path=self.reader.study_path, study_title="")
copy.load_from_file()
os.unlink(loaded_study.get_full_path())
diff --git a/tests/test_tmp_edit_window.py b/tests/test_tmp_edit_window.py
index 987ec00..e53291e 100644
--- a/tests/test_tmp_edit_window.py
+++ b/tests/test_tmp_edit_window.py
@@ -1,3 +1,9 @@
+# Skip this module if NO_WX in env
+import pytest
+import os
+if os.getenv('NO_WX'):
+ pytest.skip('Skipping module because wxPython is not available', allow_module_level=True)
+
import unittest
import os
import tempfile
diff --git a/tests/test_warning_handler.py b/tests/test_warning_handler.py
index 9976ffa..7dcbfe3 100644
--- a/tests/test_warning_handler.py
+++ b/tests/test_warning_handler.py
@@ -1,4 +1,11 @@
+# Skip this module if NO_WX in env
+import pytest
+import os
+if os.getenv('NO_WX'):
+ pytest.skip('Skipping module because wxPython is not available', allow_module_level=True)
+
import logging
+from unittest import mock
import unittest
from kvalikirstu2 import argument_parser
@@ -9,22 +16,22 @@ class TestWarningHandler(unittest.TestCase):
settings.set_up_gui_settings([])
self.logger = logging.getLogger('kvalikirstu2.testlogger')
- @unittest.mock.patch('wx.MessageBox')
+ @mock.patch('wx.MessageBox')
def test_logger_opens_warning_message(self, mock):
self.logger.warning('test')
self.assertTrue(mock.called)
- @unittest.mock.patch('wx.MessageBox')
+ @mock.patch('wx.MessageBox')
def test_logger_opens_error_message(self, mock):
self.logger.error('test')
self.assertFalse(mock.called)
- @unittest.mock.patch('wx.MessageBox')
+ @mock.patch('wx.MessageBox')
def test_info(self, mock):
self.logger.info('test')
self.assertFalse(mock.called)
- @unittest.mock.patch('wx.MessageBox')
+ @mock.patch('wx.MessageBox')
def test_debug(self, mock):
self.logger.info('test')
self.assertFalse(mock.called)
diff --git a/tox.ini b/tox.ini
index b23ea8b..a8af8e8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,8 +1,31 @@
+[tox]
+envlist = py38, py310, py311
+
[testenv]
-deps = -rrequirements_test.txt
+deps = -rrequirements.txt
+ -rrequirements_test.txt
+commands = pytest
+
+[testenv:with-coverage]
+deps = {[testenv]deps}
+ coverage==7.2.3
+commands = pytest --cov=kvalikirstu2
+ coverage xml
+ coverage report
+
+[testenv:jenkins]
+setenv =
+ NO_WX = 1
+ NO_ODT = 1
+deps = {[testenv]deps}
+commands = {[testenv]commands}
-commands = pytest --cov --cov-append --cov-report=term-missing {posargs}
- pytest --cov --cov-report xml {posargs}
+[testenv:jenkins-with-coverage]
+setenv =
+ NO_WX = 1
+ NO_ODT = 1
+deps = {[testenv:with-coverage]deps}
+commands = {[testenv:with-coverage]commands}
[flake8]
max-line-length = 120
\ No newline at end of file
--
GitLab