From 981ba15f2621fb4c242cb26422198bd0cd89206e Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Mon, 12 Aug 2024 10:46:38 +0200 Subject: [PATCH 1/4] add models-ng (C++ modules), Rocky8 Dockerfile and update rpms.sh --- .gitmodules | 3 +++ docker/rocky8/Dockerfile | 20 ++++++++++++++++++ modules-ng | 1 + rpms.sh | 44 +++++++++++++++++++++++++++------------- 4 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 docker/rocky8/Dockerfile create mode 160000 modules-ng diff --git a/.gitmodules b/.gitmodules index c1a8669..532387e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "detectors"] path = detectors url = https://github.com/CESNET/Nemea-Detectors +[submodule "modules-ng"] + path = modules-ng + url = https://github.com/CESNET/nemea-modules-ng diff --git a/docker/rocky8/Dockerfile b/docker/rocky8/Dockerfile new file mode 100644 index 0000000..187eaf5 --- /dev/null +++ b/docker/rocky8/Dockerfile @@ -0,0 +1,20 @@ +FROM rockylinux:8-minimal +RUN microdnf install dnf +RUN dnf -y install epel-release +RUN curl -s 'https://copr.fedorainfracloud.org/coprs/g/CESNET/NEMEA/repo/epel-8/group_CESNET-NEMEA-epel-8.repo' > /etc/yum.repos.d/group_CESNET-NEMEA-epel-8.repo +#RUN dnf -y update +RUN dnf -y --enablerepo=devel --enablerepo=powertools install epel-rpm-macros \ + python3-rpm-macros python-rpm-macros python-srpm-macros rpm-build git which \ + doxygen bison flex libcurl-devel lua-devel bzip2-devel libpcap-devel \ + openssl-devel libxml2-devel libidn-devel make automake autoconf \ + pkgconfig gcc gcc-c++ libtool cmake \ + python3-devel python3-setuptools python3-pip python3-wheel \ + python38 python38-devel python38-pip python38-wheel python38-setuptools \ + python39 python39-devel python39-pip python39-wheel python39-setuptools \ + python3-jinja2 python3-ply python3-pynspect python3-yaml + + # python3-nemea-pytrap +RUN dnf clean all + +CMD ["/bin/bash"] + diff --git a/modules-ng b/modules-ng new file mode 160000 index 0000000..9d81cf1 --- /dev/null +++ b/modules-ng @@ -0,0 +1 @@ +Subproject commit 9d81cf1b59a8ba0cd357a1b0687d2d057106c471 diff --git a/rpms.sh b/rpms.sh index a06cbdf..b128b2a 100755 --- a/rpms.sh +++ b/rpms.sh @@ -36,6 +36,14 @@ #set -x +if [ "$#" -eq 1 ]; then + chuser="$1" +else + echo "For non-interactive mode run with username as argument, e.g.:" + echo " $0 builder" + echo "where builder is a non-root user for building RPMs." +fi + if [ -x "`which dnf`" ]; then export pkginst=dnf elif [ -x "`which yum`" ]; then @@ -45,23 +53,28 @@ else exit 1 fi -echo "Warning: You must have 'rpmbuild' in order to generate RPM package." -echo "If you want to abort this script, press CTRL+C (i.e. send SIGINT signal)" -sleep 5 +if [ -z "$chuser" ]; then + echo "Warning: You must have 'rpmbuild' in order to generate RPM package." + echo "If you want to abort this script, press CTRL+C (i.e. send SIGINT signal)" + sleep 5 -if [ "x`whoami`" != xroot ]; then - echo "Run this script as root, since it must install RPM packages continuously" - exit 1 -fi + if [ "x`whoami`" != xroot ]; then + echo "Run this script as root, since it must install RPM packages continuously" + exit 1 + fi -read -p "Enter the name of user who will compile packages: " chuser + read -p "Enter the name of user who will compile packages: " chuser -read -p "Are You sure You want to continue? [yn]" -n1 ans + read -p "Are You sure You want to continue? [yn]" -n1 ans -if [ "x$ans" != xy ]; then - exit 0 + if [ "x$ans" != xy ]; then + exit 0 + fi fi +./bootstrap.sh +./configure -q + echo "Remove previously installed packages" $pkginst remove -q -y libtrap\* unirec\* nemea\* $pkginst install -q -y libnf-devel libpcap-devel libidn-devel bison flex @@ -88,9 +101,12 @@ export chuser ) su $chuser -p -c "./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q" ( - cd python - su $chuser -p -c "make && make rpm" - $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + cd pytrap + su $chuser -p -c "make rpm" + $pkginst install -y -q $(find \( -name '*.rpm' -o -name '*64.rpm' \)) + su $chuser -p -c "python3.6 setup.py bdist_wheel" + su $chuser -p -c "python3.8 setup.py bdist_wheel" + su $chuser -p -c "python3.9 setup.py bdist_wheel" ) ( cd pycommon From b282a7360ee4ef5ced45ab5223c5c5797511c811 Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Tue, 13 Aug 2024 00:39:34 +0200 Subject: [PATCH 2/4] update submodules, update docker&rpms.sh, add mock configs --- docker/rocky8/Dockerfile | 4 ++-- docker/rocky8/README.md | 14 ++++++++++++++ docker/rocky8/build.sh | 25 +++++++++++++++++++++++++ generate-rpm.sh | 3 ++- mock/README.md | 12 ++++++++++++ mock/rocky+epel+nemea-8-x86_64.cfg | 20 ++++++++++++++++++++ mock/rocky+epel+nemea-9-x86_64.cfg | 19 +++++++++++++++++++ modules | 2 +- nemea-framework | 2 +- nemea-supervisor | 2 +- rpms.sh | 18 +++++++++++++++--- 11 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 docker/rocky8/README.md create mode 100755 docker/rocky8/build.sh create mode 100644 mock/README.md create mode 100644 mock/rocky+epel+nemea-8-x86_64.cfg create mode 100644 mock/rocky+epel+nemea-9-x86_64.cfg diff --git a/docker/rocky8/Dockerfile b/docker/rocky8/Dockerfile index 187eaf5..af06061 100644 --- a/docker/rocky8/Dockerfile +++ b/docker/rocky8/Dockerfile @@ -6,8 +6,8 @@ RUN curl -s 'https://copr.fedorainfracloud.org/coprs/g/CESNET/NEMEA/repo/epel-8/ RUN dnf -y --enablerepo=devel --enablerepo=powertools install epel-rpm-macros \ python3-rpm-macros python-rpm-macros python-srpm-macros rpm-build git which \ doxygen bison flex libcurl-devel lua-devel bzip2-devel libpcap-devel \ - openssl-devel libxml2-devel libidn-devel make automake autoconf \ - pkgconfig gcc gcc-c++ libtool cmake \ + ncurses-devel fuse3-devel openssl-devel libxml2-devel libidn-devel make automake \ + autoconf pkgconfig gcc gcc-c++ libtool cmake \ python3-devel python3-setuptools python3-pip python3-wheel \ python38 python38-devel python38-pip python38-wheel python38-setuptools \ python39 python39-devel python39-pip python39-wheel python39-setuptools \ diff --git a/docker/rocky8/README.md b/docker/rocky8/README.md new file mode 100644 index 0000000..b83210c --- /dev/null +++ b/docker/rocky8/README.md @@ -0,0 +1,14 @@ +# Explanation + +This directory contains `Dockerfile` to build an image based on RockyLinux8 with +all required dependencies installed. + +The image can be used to create a container and build all NEMEA RPM packages +and Python wheels. + +To do this, just run `build.sh`, and the results should be in `rpms/` an `wheels/`. + +# Requirements + +* podman or docker + diff --git a/docker/rocky8/build.sh b/docker/rocky8/build.sh new file mode 100755 index 0000000..9926eb6 --- /dev/null +++ b/docker/rocky8/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +CONT_BIN="`which podman 2>/dev/null`" +if [ -z "$CONT_BIN" ]; then + CONT_BIN="`which docker 2>/dev/null`" +fi +if [ -z "$CONT_BIN" ]; then + echo "Missing podman or docker." + exit 1 +fi + +"$CONT_BIN" build --tag rl8docker:latest . +"$CONT_BIN" run -ti --rm --name rl8docker -d rl8docker:latest +"$CONT_BIN" cp ../../ rl8docker:/nemea +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ./rpms.sh root' +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ls -l rpms' +"$CONT_BIN" cp rl8docker:/nemea/rpms rpms +"$CONT_BIN" cp rl8docker:/nemea/nemea-framework/pytrap/dist/ wheels/ + +# This will remove the container: +"$CONT_BIN" stop rl8docker + +# This would remove the image: +# "$CONT_BIN" rmi rl8docker:latest + diff --git a/generate-rpm.sh b/generate-rpm.sh index 7ea9b83..41cd86e 100755 --- a/generate-rpm.sh +++ b/generate-rpm.sh @@ -35,5 +35,6 @@ # if advised of the possibility of such damage. pwd -./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q && make -j4 && make rpm +git submodule init; git submodule update; +./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q && make rpm diff --git a/mock/README.md b/mock/README.md new file mode 100644 index 0000000..30cacb3 --- /dev/null +++ b/mock/README.md @@ -0,0 +1,12 @@ +# Explanation + +This directory contains `mock` configuration files, which include @CESNET/NEMEA +copr repo with dependencies. + +Example how to rebuild a package for, e.g., libtrap for RockyLinux8 is as follows: + +``` +mock -r rocky+epel+nemea-8-x86_64.cfg path/to/srpms/libtrap-1.18.1-1.src.rpm +``` + +See the output of the command to find the results and logs. diff --git a/mock/rocky+epel+nemea-8-x86_64.cfg b/mock/rocky+epel+nemea-8-x86_64.cfg new file mode 100644 index 0000000..debc1de --- /dev/null +++ b/mock/rocky+epel+nemea-8-x86_64.cfg @@ -0,0 +1,20 @@ +include('rocky+epel-8-x86_64.cfg') + +config_opts['dnf.conf'] += """ +[copr:copr.fedorainfracloud.org:group_CESNET:NEMEA] +name=Copr repo for NEMEA owned by @CESNET +baseurl=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/epel-8-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 + +""" + +config_opts['description'] = 'Rocky Linux 8 + EPEL + NEMEA' +config_opts['root'] = "rocky+epel+nemea-8-{{ target_arch }}" + + diff --git a/mock/rocky+epel+nemea-9-x86_64.cfg b/mock/rocky+epel+nemea-9-x86_64.cfg new file mode 100644 index 0000000..0112df0 --- /dev/null +++ b/mock/rocky+epel+nemea-9-x86_64.cfg @@ -0,0 +1,19 @@ +include('rocky+epel-9-x86_64.cfg') + +config_opts['dnf.conf'] += """ +[copr:copr.fedorainfracloud.org:group_CESNET:NEMEA] +name=Copr repo for NEMEA owned by @CESNET +baseurl=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/epel-9-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 + +""" + +config_opts['description'] = 'Rocky Linux 9 + EPEL + NEMEA' +config_opts['root'] = "rocky+epel+nemea-9-{{ target_arch }}" + diff --git a/modules b/modules index c167667..1873c23 160000 --- a/modules +++ b/modules @@ -1 +1 @@ -Subproject commit c16766702209b4b899cf1ee101ae97c08e0e0b05 +Subproject commit 1873c230e84d07e617427ba2ef65fcc46574b0a1 diff --git a/nemea-framework b/nemea-framework index a554edf..a8d19d1 160000 --- a/nemea-framework +++ b/nemea-framework @@ -1 +1 @@ -Subproject commit a554edfb3bb834218ba4a1268188a641cd07ae6f +Subproject commit a8d19d1e66dd8e794c8d83f91936bb61f1b963df diff --git a/nemea-supervisor b/nemea-supervisor index 89afda1..cd0e74c 160000 --- a/nemea-supervisor +++ b/nemea-supervisor @@ -1 +1 @@ -Subproject commit 89afda171fc119db253235f7a8e0c71614296dec +Subproject commit cd0e74cbc4450e918e011d5b202fc5aa8809188c diff --git a/rpms.sh b/rpms.sh index b128b2a..99e8358 100755 --- a/rpms.sh +++ b/rpms.sh @@ -103,16 +103,19 @@ export chuser ( cd pytrap su $chuser -p -c "make rpm" - $pkginst install -y -q $(find \( -name '*.rpm' -o -name '*64.rpm' \)) + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) su $chuser -p -c "python3.6 setup.py bdist_wheel" su $chuser -p -c "python3.8 setup.py bdist_wheel" su $chuser -p -c "python3.9 setup.py bdist_wheel" ) ( cd pycommon - su $chuser -p -c "make && make rpm" + su $chuser -p -c "make rpm" + su $chuser -p -c "python3.6 setup.py bdist_wheel" $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) ) + su $chuser -p -c "make rpm" + $pkginst install -y -q $(find -name 'nemea-framework-*noarch.rpm') ) ( cd modules @@ -124,6 +127,11 @@ export chuser su $chuser -p -c "$topdir/generate-rpm.sh" $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) ) +( + cd modules-ng + su $chuser -p -c "cmake . && make rpm" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) +) ( cd nemea-supervisor su $chuser -p -c "$topdir/generate-rpm.sh" @@ -134,6 +142,10 @@ su $chuser -p -c "$topdir/bootstrap.sh >/dev/null 2>/dev/null&& $topdir/configur mkdir -p "`pwd`/RPMBUILD" rpmbuild -ba nemea.spec --define "_topdir `pwd`/RPMBUILD" mkdir -p "`pwd`/rpms" -find -name *.rpm -not -path "./rpms/*" -exec mv {} rpms/ \; +find -name '*.rpm' -not -path "./rpms/*" -exec mv {} rpms/ \; chown -R $chuser rpms/ +mkdir -p "`pwd`/wheels" +find -name '*.whl' -not -path "./wheels/*" -exec mv {} wheels/ \; + + From 33b22822141a06df9288cd593353fd6872ff9a32 Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Fri, 13 Sep 2024 21:05:03 +0200 Subject: [PATCH 3/4] docker: build doc --- docker/rocky8/Dockerfile | 3 ++- docker/rocky8/build.sh | 11 +++++++++-- modules | 2 +- nemea-framework | 2 +- nemea-supervisor | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docker/rocky8/Dockerfile b/docker/rocky8/Dockerfile index af06061..f66cce4 100644 --- a/docker/rocky8/Dockerfile +++ b/docker/rocky8/Dockerfile @@ -11,7 +11,8 @@ RUN dnf -y --enablerepo=devel --enablerepo=powertools install epel-rpm-macros \ python3-devel python3-setuptools python3-pip python3-wheel \ python38 python38-devel python38-pip python38-wheel python38-setuptools \ python39 python39-devel python39-pip python39-wheel python39-setuptools \ - python3-jinja2 python3-ply python3-pynspect python3-yaml + python3-jinja2 python3-ply python3-pynspect python3-yaml \ + python3-sphinx # python3-nemea-pytrap RUN dnf clean all diff --git a/docker/rocky8/build.sh b/docker/rocky8/build.sh index 9926eb6..d2cf65d 100755 --- a/docker/rocky8/build.sh +++ b/docker/rocky8/build.sh @@ -13,9 +13,16 @@ fi "$CONT_BIN" run -ti --rm --name rl8docker -d rl8docker:latest "$CONT_BIN" cp ../../ rl8docker:/nemea "$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ./rpms.sh root' -"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ls -l rpms' +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea/nemea-framework/; for i in libtrap unirec common pytrap; do ( cd $i; make doc; ) done; cd /nemea; mkdir doc; \ +mv nemea-framework/libtrap/doc/doxygen/html doc/libtrap; \ +mv nemea-framework/libtrap/doc/devel/html doc/libtrap-devel; \ +mv nemea-framework/unirec/doc/html doc/unirec; \ +mv nemea-framework/common/doc/html doc/nemea-common; \ +mv nemea-framework/pytrap/docs/html doc/pytrap;' +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ls -l rpms doc' "$CONT_BIN" cp rl8docker:/nemea/rpms rpms -"$CONT_BIN" cp rl8docker:/nemea/nemea-framework/pytrap/dist/ wheels/ +"$CONT_BIN" cp rl8docker:/nemea/nemea-framework/pytrap/dist wheels +"$CONT_BIN" cp rl8docker:/nemea/doc doc # This will remove the container: "$CONT_BIN" stop rl8docker diff --git a/modules b/modules index 1873c23..c167667 160000 --- a/modules +++ b/modules @@ -1 +1 @@ -Subproject commit 1873c230e84d07e617427ba2ef65fcc46574b0a1 +Subproject commit c16766702209b4b899cf1ee101ae97c08e0e0b05 diff --git a/nemea-framework b/nemea-framework index a8d19d1..54a727d 160000 --- a/nemea-framework +++ b/nemea-framework @@ -1 +1 @@ -Subproject commit a8d19d1e66dd8e794c8d83f91936bb61f1b963df +Subproject commit 54a727dbd95767cc3a013edc6c66612a81687409 diff --git a/nemea-supervisor b/nemea-supervisor index cd0e74c..89afda1 160000 --- a/nemea-supervisor +++ b/nemea-supervisor @@ -1 +1 @@ -Subproject commit cd0e74cbc4450e918e011d5b202fc5aa8809188c +Subproject commit 89afda171fc119db253235f7a8e0c71614296dec From 02904f9acd0bef82c07ecb77da817a952d22019e Mon Sep 17 00:00:00 2001 From: Tomas Cejka Date: Sat, 26 Oct 2024 20:40:32 +0200 Subject: [PATCH 4/4] vagrant: update, migrate to oraclelinux9 with ansible installation --- vagrant/CentOS7/README.md | 29 --- vagrant/CentOS7/Vagrantfile | 54 ---- vagrant/Fedora22/README.md | 28 -- vagrant/Fedora22/Vagrantfile | 72 ------ vagrant/OracleLinux9/README.md | 29 +++ vagrant/OracleLinux9/Vagrantfile | 40 +++ vagrant/README.md | 7 +- vagrant/ScientificLinux6/README.md | 28 -- vagrant/ScientificLinux6/Vagrantfile | 76 ------ vagrant/ansible/inventory/hosts | 3 + vagrant/ansible/nemea-vm.yml | 9 + .../ansible/roles/common_tools/meta/main.yml | 4 + .../ansible/roles/common_tools/tasks/main.yml | 21 ++ vagrant/ansible/roles/configs/files/eth0.conf | 11 + .../roles/configs/files/ipfixcol2-startup.xml | 59 +++++ .../ansible/roles/configs/files/logger.sup | 35 +++ vagrant/ansible/roles/configs/files/motd | 6 + vagrant/ansible/roles/configs/tasks/main.yml | 51 ++++ .../roles/ipfixcol2/files/ipfixcol2.service | 15 ++ vagrant/ansible/roles/ipfixcol2/meta/main.yml | 4 + .../ansible/roles/ipfixcol2/tasks/main.yml | 17 ++ .../ansible/roles/ipfixprobe/meta/main.yml | 4 + .../ansible/roles/ipfixprobe/tasks/main.yml | 6 + vagrant/ansible/roles/nemea/meta/main.yml | 4 + vagrant/ansible/roles/nemea/tasks/main.yml | 7 + .../roles/rpm_repositories/tasks/main.yml | 17 ++ vagrant/common/Makefile.am | 2 - vagrant/common/nemea-dashboard.rb | 242 ------------------ 28 files changed, 346 insertions(+), 534 deletions(-) delete mode 100644 vagrant/CentOS7/README.md delete mode 100644 vagrant/CentOS7/Vagrantfile delete mode 100644 vagrant/Fedora22/README.md delete mode 100644 vagrant/Fedora22/Vagrantfile create mode 100644 vagrant/OracleLinux9/README.md create mode 100644 vagrant/OracleLinux9/Vagrantfile delete mode 100644 vagrant/ScientificLinux6/README.md delete mode 100644 vagrant/ScientificLinux6/Vagrantfile create mode 100644 vagrant/ansible/inventory/hosts create mode 100644 vagrant/ansible/nemea-vm.yml create mode 100644 vagrant/ansible/roles/common_tools/meta/main.yml create mode 100644 vagrant/ansible/roles/common_tools/tasks/main.yml create mode 100644 vagrant/ansible/roles/configs/files/eth0.conf create mode 100644 vagrant/ansible/roles/configs/files/ipfixcol2-startup.xml create mode 100644 vagrant/ansible/roles/configs/files/logger.sup create mode 100644 vagrant/ansible/roles/configs/files/motd create mode 100644 vagrant/ansible/roles/configs/tasks/main.yml create mode 100644 vagrant/ansible/roles/ipfixcol2/files/ipfixcol2.service create mode 100644 vagrant/ansible/roles/ipfixcol2/meta/main.yml create mode 100644 vagrant/ansible/roles/ipfixcol2/tasks/main.yml create mode 100644 vagrant/ansible/roles/ipfixprobe/meta/main.yml create mode 100644 vagrant/ansible/roles/ipfixprobe/tasks/main.yml create mode 100644 vagrant/ansible/roles/nemea/meta/main.yml create mode 100644 vagrant/ansible/roles/nemea/tasks/main.yml create mode 100644 vagrant/ansible/roles/rpm_repositories/tasks/main.yml delete mode 100644 vagrant/common/Makefile.am delete mode 100644 vagrant/common/nemea-dashboard.rb diff --git a/vagrant/CentOS7/README.md b/vagrant/CentOS7/README.md deleted file mode 100644 index 88f2534..0000000 --- a/vagrant/CentOS7/README.md +++ /dev/null @@ -1,29 +0,0 @@ -Installation of NEMEA using Vagrant on CentOS7 -============================================== - -If you do not have a vagrant box for CentOS7 box yet, use: - -``` -vagrant box add centos/7 --provider=virtualbox -``` - -Installation - Final Step -========================= - -Start the VM Installation using (this will take few minutes): -``` -vagrant up -``` - -Once the installation is complete, SSH into the VM: -``` -vagrant ssh -``` - -Content of VM -============= - -* all NEMEA packages installed from RPM from homeproj.cesnet.cz -* NEMEA-Dashboard -* ipfixcol with ipfixcol-unirec-output - diff --git a/vagrant/CentOS7/Vagrantfile b/vagrant/CentOS7/Vagrantfile deleted file mode 100644 index a23f207..0000000 --- a/vagrant/CentOS7/Vagrantfile +++ /dev/null @@ -1,54 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -load '../common/nemea-dashboard.rb' - -$tools = <