openssl:base:update110

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision		 Previous revision
openssl:base:update110 [2018/06/13 10:48] – Add analysis of MKLINK generation BernardSpilopenssl:base:update110 [2018/07/16 03:58] (current) – GordonTetlow
Line 1: Line 1:
-Process to update %%OpenSSL%% in base+Process to update %%OpenSSL%% in base\\  
 +(for 1.1.1-pre8 WIP see [[OpenSSL/Base/Update111]]) 
 + 
 +====== Summary ====== 
 + 
 +For the impatient... 
 + 
 +<code> 
 +svn checkout https://svn.FreeBSD.org/base/head /usr/src 
 +fetch -o ~/patch-D15791 'https://reviews.freebsd.org/D15791?download=true' 
 +fetch -o ~/openssl-1.1.0h.tar.gz https://www.openssl.org/source/openssl-1.1.0h.tar.gz 
 +cd /usr/src 
 +patch -p0 < ~/patch-D15791 
 +mv /usr/src/crypto/openssl /usr/src/crypto/openssl-1.0.2 
 +tar xf ~/openssl-1.1.0h.tar.gz -C /usr/src/crypto 
 +mv /usr/src/crypto/openssl-1.1.0h /usr/src/crypto/openssl 
 +cd /usr/src/secure/lib/libcrypto 
 +pkg install perl5 
 +rm amd64/* 
 +mv /usr/obj/usr/src/amd64.amd64/secure/lib/libcrypto/*.S amd64/ 
 +make man-makefile-update 
 +rm man/* 
 +make man-update 
 +cd ../libssl 
 +make man-makefile-update 
 +rm man/* 
 +make man-update 
 +</code>
  
 ====== Preliminary steps ====== ====== Preliminary steps ======
Line 52: Line 79:
 $ pkg install perl5 $ pkg install perl5
 ... ...
 +</code>
 +
 +====== Generate .h files ======
 +
 +Complication here, as there are different ''%%opensslconf.h%%'' files for various arch. For now I generated an opensslconf.h like so
 +
 +<code>
 +$ uname -a
 +FreeBSD freebsd12 12.0-CURRENT FreeBSD 12.0-CURRENT #0 r334983: Tue Jun 12 01:42:40 UTC 2018     root@releng3.nyi.freebsd.org:/usr/obj/usr/src/amd64.amd64/sys/GENERIC  amd64
 +$ ./config -v
 +Operating system: amd64-whatever-freebsd
 +Configuring for BSD-x86_64
 +/usr/local/bin/perl5 ./Configure BSD-x86_64
 +Configuring OpenSSL version 1.1.0h (0x1010008fL)
 +    no-asan         [default]  OPENSSL_NO_ASAN
 +    no-crypto-mdebug [default]  OPENSSL_NO_CRYPTO_MDEBUG
 +    no-crypto-mdebug-backtrace [default]  OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
 +    no-ec_nistp_64_gcc_128 [default]  OPENSSL_NO_EC_NISTP_64_GCC_128
 +    no-egd          [default]  OPENSSL_NO_EGD
 +    no-fuzz-afl     [default]  OPENSSL_NO_FUZZ_AFL
 +    no-fuzz-libfuzzer [default]  OPENSSL_NO_FUZZ_LIBFUZZER
 +    no-heartbeats   [default]  OPENSSL_NO_HEARTBEATS
 +    no-md2          [default]  OPENSSL_NO_MD2 (skip dir)
 +    no-msan         [default]  OPENSSL_NO_MSAN
 +    no-rc5          [default]  OPENSSL_NO_RC5 (skip dir)
 +    no-sctp         [default]  OPENSSL_NO_SCTP
 +    no-ssl-trace    [default]  OPENSSL_NO_SSL_TRACE
 +    no-ssl3         [default]  OPENSSL_NO_SSL3
 +    no-ssl3-method  [default]  OPENSSL_NO_SSL3_METHOD
 +    no-ubsan        [default]  OPENSSL_NO_UBSAN
 +    no-unit-test    [default]  OPENSSL_NO_UNIT_TEST
 +    no-weak-ssl-ciphers [default]  OPENSSL_NO_WEAK_SSL_CIPHERS
 +    no-zlib         [default]
 +    no-zlib-dynamic [default]
 +Configuring for BSD-x86_64
 +CC            =cc
 +CFLAG         =-DL_ENDIAN -Wall -O3 -pthread -D_THREAD_SAFE -D_REENTRANT
 +SHARED_CFLAG  =-fPIC
 +DEFINES       =DSO_DLFCN HAVE_DLFCN_H NDEBUG OPENSSL_THREADS OPENSSL_NO_STATIC_ENGINE OPENSSL_PIC OPENSSL_IA32_SSE2 OPENSSL_BN_ASM_MONT OPENSSL_BN_ASM_MONT5 OPENSSL_BN_ASM_GF2m SHA1_ASM SHA256_ASM SHA512_ASM RC4_ASM MD5_ASM AES_ASM VPAES_ASM BSAES_ASM GHASH_ASM ECP_NISTZ256_ASM PADLOCK_ASM POLY1305_ASM
 +LFLAG         =
 +PLIB_LFLAG    =
 +EX_LIBS       =
 +APPS_OBJ      =
 +CPUID_OBJ     =x86_64cpuid.o
 +UPLINK_OBJ    =
 +BN_ASM        =asm/x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o
 +EC_ASM        =ecp_nistz256.o ecp_nistz256-x86_64.o
 +DES_ENC       =des_enc.o fcrypt_b.o
 +AES_ENC       =aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o
 +BF_ENC        =bf_enc.o
 +CAST_ENC      =c_enc.o
 +RC4_ENC       =rc4-x86_64.o rc4-md5-x86_64.o
 +RC5_ENC       =rc5_enc.o
 +MD5_OBJ_ASM   =md5-x86_64.o
 +SHA1_OBJ_ASM  =sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o
 +RMD160_OBJ_ASM=
 +CMLL_ENC      =cmll-x86_64.o cmll_misc.o
 +MODES_OBJ     =ghash-x86_64.o aesni-gcm-x86_64.o
 +PADLOCK_OBJ   =e_padlock-x86_64.o
 +CHACHA_ENC    =chacha-x86_64.o
 +POLY1305_OBJ  =poly1305-x86_64.o
 +BLAKE2_OBJ    =
 +PROCESSOR     =
 +RANLIB        =ranlib
 +ARFLAGS       =
 +PERL          =/usr/local/bin/perl5
 +
 +SIXTY_FOUR_BIT_LONG mode
 +
 +Configured for BSD-x86_64.
 +</code>
 +
 +**TODO: Generate proper opensslconf.h files for all archs**\\ 
 +Same applies to crypto/include/internal/bn_conf.h\\ 
 +This is a hint from the %%OpenSSL%% Makefile
 +
 +<code>
 +GENERATED_MANDATORY=crypto/include/internal/bn_conf.h crypto/include/internal/dso_conf.h include/openssl/opensslconf.h
 +</code>
 +
 +and from the make output
 +
 +<code>
 +/usr/local/bin/perl5 "-I." -Mconfigdata "util/dofile.pl"  "-oMakefile" crypto/include/internal/bn_conf.h.in > crypto/include/internal/bn_conf.h
 +/usr/local/bin/perl5 "-I." -Mconfigdata "util/dofile.pl"  "-oMakefile" crypto/include/internal/dso_conf.h.in > crypto/include/internal/dso_conf.h
 +/usr/local/bin/perl5 "-I." -Mconfigdata "util/dofile.pl"  "-oMakefile" include/openssl/opensslconf.h.in > include/openssl/opensslconf.h
 </code> </code>
  
Line 80: Line 193:
 **TODO: Verify that we have all necessary ''%%.S%%'' files**\\  **TODO: Verify that we have all necessary ''%%.S%%'' files**\\ 
 The files processed are defined in the Makefile, which is thus version-specific. The files processed are defined in the Makefile, which is thus version-specific.
 +
 +===== Update Makefile.asm .pl sources =====
 +
 +%%OpenSSL%% 1.1 added some perl files that generate ASM.
 +From ''%%${WRKSRC}/Configurations/00-base-templates.conf%%'' we can find out what %%OpenSSL%%'s current list of assembly files is per architecture.
 +Quite some additional files to be added. Chacha20/Poly1305 for all archs, some other for ARM(64).
  
 ====== Manpage generation ====== ====== Manpage generation ======
 +
 +===== TL;DR =====
 +
 +After fixing Makefile.inc ''%%MKLINK%%'' output
 +
 +<code>
 +$ make -C /usr/src/secure/lib/libcrypto man-makefile-update
 +$ make -C /usr/src/secure/lib/libcrypto man-update
 +$ make -C /usr/src/secure/lib/libssl man-makefile-update
 +$ make -C /usr/src/secure/lib/libssl man-update
 +$ make -C /usr/src/secure/usr.bin/openssl man-makefile-update
 +$ make -C /usr/src/secure/usr.bin/openssl man-update
 +</code>
 +
 +Work log:
  
 Two targets in Makefile.inc: ''%%man-update%%'' and ''%%man-makefile-update%%''. Two targets in Makefile.inc: ''%%man-update%%'' and ''%%man-makefile-update%%''.
Line 162: Line 296:
  
 Conclusion: Part of man-makefile-update that collects ''%%.pod%%'' files works, part that extracts aliases doesn't.\\  Conclusion: Part of man-makefile-update that collects ''%%.pod%%'' files works, part that extracts aliases doesn't.\\ 
-**TODO: Fix ''%%MLINKS+=%%'' part of ''%%man-makefile-update%%''** (hint: ''%%process-docs.pl%%'')+<del>**TODO: Fix ''%%MLINKS+=%%'' part of ''%%man-makefile-update%%''** (hint: ''%%process-docs.pl%%'')</del>
  
 ===== extract-names.pl and process_docs.pl ===== ===== extract-names.pl and process_docs.pl =====
Line 189: Line 323:
 This should lead to a single sym-link ''%%ASN1_ITEM_get.3 -> ASN1_ITEM_lookup.3%%'' This should lead to a single sym-link ''%%ASN1_ITEM_get.3 -> ASN1_ITEM_lookup.3%%''
  
-==== process_dcos.pl ====+==== process_docs.pl ====
  
 Looks like the part we're interested in is [[https://github.com/openssl/openssl/blob/master/util/process_docs.pl#L169|line 169 onwards]]. Looks like the part we're interested in is [[https://github.com/openssl/openssl/blob/master/util/process_docs.pl#L169|line 169 onwards]].
Line 199: Line 333:
 sed '/=head1 NAME/,/=head1 /! d;/=head1.*/d;/^\s*$/d;s/ - .*$//;s/,//g' BIO_ADDR.pod || tr ' ' '\n' sed '/=head1 NAME/,/=head1 /! d;/=head1.*/d;/^\s*$/d;s/ - .*$//;s/,//g' BIO_ADDR.pod || tr ' ' '\n'
 </code> </code>
 +
 +==== Diff for secure/lib/libcrypto/Makefile.inc ====
 +
 +Includes the changes for the version
 +
 +<code>
 +Index: Makefile.inc
 +===================================================================
 +--- Makefile.inc (revision 335034)
 ++++ Makefile.inc (working copy)
 +@@ -3,7 +3,7 @@
 + .include <bsd.own.mk>
 +
 + # OpenSSL version used for manual page generation
 +-OPENSSL_VER=   1.0.2o
 ++OPENSSL_VER=   1.1.0h
 + OPENSSL_DATE=  2018-03-27
 +
 + LCRYPTO_SRC=   ${SRCTOP}/crypto/openssl
 +@@ -106,7 +106,8 @@
 + test -f $$j || continue; \
 + fn=`basename $$j .pod`; \
 + if ! echo '${_skip}' | grep -qw "$$fn"; then \
 +-        perl ${LCRYPTO_SRC}/util/extract-names.pl < $$j | \
 ++        sed '/=head1 NAME/,/^=.*/! d;/^=.*/d;/^\s*$$/d' $$j | tr '\n' ' ' | \
 ++ sed 's/ - .*$$//;s/,/ /g;s/  / /g' | tr ' ' '\n' | \
 +   awk "/^$$fn\$$/ { next; } \
 +   { print \"MLINKS+= $$fn.${_sec} \" \$$1 \".${_sec}\" }"; \
 + fi; \
 +</code>
 +
 +====== Update secure/libcrypto/Makefile ======
 +
 +Sources to be checked is ''%%crypto/*/build.info%%'' which defines the sources to be used. \\ 
 +Helper to extract SRCS+= from a dir
 +
 +<code>
 +cd /usr/src/crypto/openssl/crypto
 +for file in */build.info ; do
 +   echo -e "\n$file"
 +   sed -E '/^SOURCE/,/^($|INCLUDE.*)/! d;/^INCLUDE.*/d;s/^SOURCE.*=//;s/\\//g;s/\{.*\}//g;s/[[:space:]]+/ /g;s/^ //;s/ $//;/^$/d' $file \
 +| tr ' ' '\n' | sort | tr '\n' ' ' | fold -s -w70 | \                                                                                                                                                                                  sed 's/$/\\/;s/^/ /;1s/^/SRCS+=/;$s/ \\//'                                                                                                                                                                                   done
 +</code>
 +
 +copy-paste.
  
  • openssl/base/update110.1528886902.txt.gz
  • Last modified: 2018/06/13 10:48
  • by BernardSpil