diff --git a/.gitmodules b/.gitmodules index c1a8669..532387e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "detectors"] path = detectors url = https://github.com/CESNET/Nemea-Detectors +[submodule "modules-ng"] + path = modules-ng + url = https://github.com/CESNET/nemea-modules-ng diff --git a/docker/rocky8/Dockerfile b/docker/rocky8/Dockerfile new file mode 100644 index 0000000..f66cce4 --- /dev/null +++ b/docker/rocky8/Dockerfile @@ -0,0 +1,21 @@ +FROM rockylinux:8-minimal +RUN microdnf install dnf +RUN dnf -y install epel-release +RUN curl -s 'https://copr.fedorainfracloud.org/coprs/g/CESNET/NEMEA/repo/epel-8/group_CESNET-NEMEA-epel-8.repo' > /etc/yum.repos.d/group_CESNET-NEMEA-epel-8.repo +#RUN dnf -y update +RUN dnf -y --enablerepo=devel --enablerepo=powertools install epel-rpm-macros \ + python3-rpm-macros python-rpm-macros python-srpm-macros rpm-build git which \ + doxygen bison flex libcurl-devel lua-devel bzip2-devel libpcap-devel \ + ncurses-devel fuse3-devel openssl-devel libxml2-devel libidn-devel make automake \ + autoconf pkgconfig gcc gcc-c++ libtool cmake \ + python3-devel python3-setuptools python3-pip python3-wheel \ + python38 python38-devel python38-pip python38-wheel python38-setuptools \ + python39 python39-devel python39-pip python39-wheel python39-setuptools \ + python3-jinja2 python3-ply python3-pynspect python3-yaml \ + python3-sphinx + + # python3-nemea-pytrap +RUN dnf clean all + +CMD ["/bin/bash"] + diff --git a/docker/rocky8/README.md b/docker/rocky8/README.md new file mode 100644 index 0000000..b83210c --- /dev/null +++ b/docker/rocky8/README.md @@ -0,0 +1,14 @@ +# Explanation + +This directory contains `Dockerfile` to build an image based on RockyLinux8 with +all required dependencies installed. + +The image can be used to create a container and build all NEMEA RPM packages +and Python wheels. + +To do this, just run `build.sh`, and the results should be in `rpms/` an `wheels/`. + +# Requirements + +* podman or docker + diff --git a/docker/rocky8/build.sh b/docker/rocky8/build.sh new file mode 100755 index 0000000..d2cf65d --- /dev/null +++ b/docker/rocky8/build.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +CONT_BIN="`which podman 2>/dev/null`" +if [ -z "$CONT_BIN" ]; then + CONT_BIN="`which docker 2>/dev/null`" +fi +if [ -z "$CONT_BIN" ]; then + echo "Missing podman or docker." + exit 1 +fi + +"$CONT_BIN" build --tag rl8docker:latest . +"$CONT_BIN" run -ti --rm --name rl8docker -d rl8docker:latest +"$CONT_BIN" cp ../../ rl8docker:/nemea +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ./rpms.sh root' +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea/nemea-framework/; for i in libtrap unirec common pytrap; do ( cd $i; make doc; ) done; cd /nemea; mkdir doc; \ +mv nemea-framework/libtrap/doc/doxygen/html doc/libtrap; \ +mv nemea-framework/libtrap/doc/devel/html doc/libtrap-devel; \ +mv nemea-framework/unirec/doc/html doc/unirec; \ +mv nemea-framework/common/doc/html doc/nemea-common; \ +mv nemea-framework/pytrap/docs/html doc/pytrap;' +"$CONT_BIN" exec -ti rl8docker bash -c 'cd /nemea; ls -l rpms doc' +"$CONT_BIN" cp rl8docker:/nemea/rpms rpms +"$CONT_BIN" cp rl8docker:/nemea/nemea-framework/pytrap/dist wheels +"$CONT_BIN" cp rl8docker:/nemea/doc doc + +# This will remove the container: +"$CONT_BIN" stop rl8docker + +# This would remove the image: +# "$CONT_BIN" rmi rl8docker:latest + diff --git a/generate-rpm.sh b/generate-rpm.sh index 7ea9b83..41cd86e 100755 --- a/generate-rpm.sh +++ b/generate-rpm.sh @@ -35,5 +35,6 @@ # if advised of the possibility of such damage. pwd -./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q && make -j4 && make rpm +git submodule init; git submodule update; +./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q && make rpm diff --git a/mock/README.md b/mock/README.md new file mode 100644 index 0000000..30cacb3 --- /dev/null +++ b/mock/README.md @@ -0,0 +1,12 @@ +# Explanation + +This directory contains `mock` configuration files, which include @CESNET/NEMEA +copr repo with dependencies. + +Example how to rebuild a package for, e.g., libtrap for RockyLinux8 is as follows: + +``` +mock -r rocky+epel+nemea-8-x86_64.cfg path/to/srpms/libtrap-1.18.1-1.src.rpm +``` + +See the output of the command to find the results and logs. diff --git a/mock/rocky+epel+nemea-8-x86_64.cfg b/mock/rocky+epel+nemea-8-x86_64.cfg new file mode 100644 index 0000000..debc1de --- /dev/null +++ b/mock/rocky+epel+nemea-8-x86_64.cfg @@ -0,0 +1,20 @@ +include('rocky+epel-8-x86_64.cfg') + +config_opts['dnf.conf'] += """ +[copr:copr.fedorainfracloud.org:group_CESNET:NEMEA] +name=Copr repo for NEMEA owned by @CESNET +baseurl=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/epel-8-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 + +""" + +config_opts['description'] = 'Rocky Linux 8 + EPEL + NEMEA' +config_opts['root'] = "rocky+epel+nemea-8-{{ target_arch }}" + + diff --git a/mock/rocky+epel+nemea-9-x86_64.cfg b/mock/rocky+epel+nemea-9-x86_64.cfg new file mode 100644 index 0000000..0112df0 --- /dev/null +++ b/mock/rocky+epel+nemea-9-x86_64.cfg @@ -0,0 +1,19 @@ +include('rocky+epel-9-x86_64.cfg') + +config_opts['dnf.conf'] += """ +[copr:copr.fedorainfracloud.org:group_CESNET:NEMEA] +name=Copr repo for NEMEA owned by @CESNET +baseurl=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/epel-9-$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/@CESNET/NEMEA/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 + +""" + +config_opts['description'] = 'Rocky Linux 9 + EPEL + NEMEA' +config_opts['root'] = "rocky+epel+nemea-9-{{ target_arch }}" + diff --git a/modules-ng b/modules-ng new file mode 160000 index 0000000..9d81cf1 --- /dev/null +++ b/modules-ng @@ -0,0 +1 @@ +Subproject commit 9d81cf1b59a8ba0cd357a1b0687d2d057106c471 diff --git a/nemea-framework b/nemea-framework index a554edf..54a727d 160000 --- a/nemea-framework +++ b/nemea-framework @@ -1 +1 @@ -Subproject commit a554edfb3bb834218ba4a1268188a641cd07ae6f +Subproject commit 54a727dbd95767cc3a013edc6c66612a81687409 diff --git a/rpms.sh b/rpms.sh index a06cbdf..99e8358 100755 --- a/rpms.sh +++ b/rpms.sh @@ -36,6 +36,14 @@ #set -x +if [ "$#" -eq 1 ]; then + chuser="$1" +else + echo "For non-interactive mode run with username as argument, e.g.:" + echo " $0 builder" + echo "where builder is a non-root user for building RPMs." +fi + if [ -x "`which dnf`" ]; then export pkginst=dnf elif [ -x "`which yum`" ]; then @@ -45,23 +53,28 @@ else exit 1 fi -echo "Warning: You must have 'rpmbuild' in order to generate RPM package." -echo "If you want to abort this script, press CTRL+C (i.e. send SIGINT signal)" -sleep 5 +if [ -z "$chuser" ]; then + echo "Warning: You must have 'rpmbuild' in order to generate RPM package." + echo "If you want to abort this script, press CTRL+C (i.e. send SIGINT signal)" + sleep 5 -if [ "x`whoami`" != xroot ]; then - echo "Run this script as root, since it must install RPM packages continuously" - exit 1 -fi + if [ "x`whoami`" != xroot ]; then + echo "Run this script as root, since it must install RPM packages continuously" + exit 1 + fi -read -p "Enter the name of user who will compile packages: " chuser + read -p "Enter the name of user who will compile packages: " chuser -read -p "Are You sure You want to continue? [yn]" -n1 ans + read -p "Are You sure You want to continue? [yn]" -n1 ans -if [ "x$ans" != xy ]; then - exit 0 + if [ "x$ans" != xy ]; then + exit 0 + fi fi +./bootstrap.sh +./configure -q + echo "Remove previously installed packages" $pkginst remove -q -y libtrap\* unirec\* nemea\* $pkginst install -q -y libnf-devel libpcap-devel libidn-devel bison flex @@ -88,15 +101,21 @@ export chuser ) su $chuser -p -c "./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q" ( - cd python - su $chuser -p -c "make && make rpm" + cd pytrap + su $chuser -p -c "make rpm" $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + su $chuser -p -c "python3.6 setup.py bdist_wheel" + su $chuser -p -c "python3.8 setup.py bdist_wheel" + su $chuser -p -c "python3.9 setup.py bdist_wheel" ) ( cd pycommon - su $chuser -p -c "make && make rpm" + su $chuser -p -c "make rpm" + su $chuser -p -c "python3.6 setup.py bdist_wheel" $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) ) + su $chuser -p -c "make rpm" + $pkginst install -y -q $(find -name 'nemea-framework-*noarch.rpm') ) ( cd modules @@ -108,6 +127,11 @@ export chuser su $chuser -p -c "$topdir/generate-rpm.sh" $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) ) +( + cd modules-ng + su $chuser -p -c "cmake . && make rpm" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) +) ( cd nemea-supervisor su $chuser -p -c "$topdir/generate-rpm.sh" @@ -118,6 +142,10 @@ su $chuser -p -c "$topdir/bootstrap.sh >/dev/null 2>/dev/null&& $topdir/configur mkdir -p "`pwd`/RPMBUILD" rpmbuild -ba nemea.spec --define "_topdir `pwd`/RPMBUILD" mkdir -p "`pwd`/rpms" -find -name *.rpm -not -path "./rpms/*" -exec mv {} rpms/ \; +find -name '*.rpm' -not -path "./rpms/*" -exec mv {} rpms/ \; chown -R $chuser rpms/ +mkdir -p "`pwd`/wheels" +find -name '*.whl' -not -path "./wheels/*" -exec mv {} wheels/ \; + + diff --git a/vagrant/CentOS7/README.md b/vagrant/CentOS7/README.md deleted file mode 100644 index 88f2534..0000000 --- a/vagrant/CentOS7/README.md +++ /dev/null @@ -1,29 +0,0 @@ -Installation of NEMEA using Vagrant on CentOS7 -============================================== - -If you do not have a vagrant box for CentOS7 box yet, use: - -``` -vagrant box add centos/7 --provider=virtualbox -``` - -Installation - Final Step -========================= - -Start the VM Installation using (this will take few minutes): -``` -vagrant up -``` - -Once the installation is complete, SSH into the VM: -``` -vagrant ssh -``` - -Content of VM -============= - -* all NEMEA packages installed from RPM from homeproj.cesnet.cz -* NEMEA-Dashboard -* ipfixcol with ipfixcol-unirec-output - diff --git a/vagrant/CentOS7/Vagrantfile b/vagrant/CentOS7/Vagrantfile deleted file mode 100644 index a23f207..0000000 --- a/vagrant/CentOS7/Vagrantfile +++ /dev/null @@ -1,54 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -load '../common/nemea-dashboard.rb' - -$tools = <