andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

georep: python2 to python3 compat - syscalls

Browse Source 
1. ctypes/syscalls
    A) arguments is expected to be encoded
    B) Raw conversion of return value from bytearray into string
2. struct pack/unpack - Raw converstion of string to bytearray
3. basestring -> str

Updates: #411
Change-Id: I80f939adcdec0ed0022c87c0b76d057ad5559e5a
Signed-off-by: Kotresh HR <khiremat@redhat.com>
This commit is contained in:
Kotresh HR 2018-10-03 00:45:09 -04:00 committed by Amar Tumballi
parent 860a990811
commit fb6e8d0d0c
7 changed files with 88 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
geo-replication/syncdaemon/Makefile.am
View File

@ -3,6 +3,6 @@ syncdaemondir = $(GLUSTERFS_LIBEXECDIR)/python/syncdaemon
syncdaemon_PYTHON = rconf.py gsyncd.py __init__.py master.py README.md repce.py \
resource.py syncdutils.py monitor.py libcxattr.py gsyncdconfig.py \
libgfchangelog.py changelogagent.py gsyncdstatus.py conf.py logutils.py \
subcmds.py argsupgrade.py
subcmds.py argsupgrade.py py2py3.py
CLEANFILES =

17
geo-replication/syncdaemon/libcxattr.py
View File

@ -10,6 +10,8 @@
import os
from ctypes import CDLL, create_string_buffer, get_errno
import py2py3
from py2py3 import bytearray_to_str
class Xattr(object):
@ -38,20 +40,23 @@ class Xattr(object):
@classmethod
def _query_xattr(cls, path, siz, syscall, *a):
if siz:
buf = create_string_buffer('\0' * siz)
buf = create_string_buffer(b'\0' * siz)
else:
buf = None
ret = getattr(cls.libc, syscall)(*((path,) + a + (buf, siz)))
if ret == -1:
cls.raise_oserr()
if siz:
return buf.raw[:ret]
# py2 and py3 compatibility. Convert bytes array
# to string
result = bytearray_to_str(buf.raw)
return result[:ret]
else:
return ret
@classmethod
def lgetxattr(cls, path, attr, siz=0):
return cls._query_xattr(path, siz, 'lgetxattr', attr)
return cls._query_xattr(path.encode(), siz, 'lgetxattr', attr.encode())
@classmethod
def lgetxattr_buf(cls, path, attr):
@ -65,7 +70,7 @@ class Xattr(object):
@classmethod
def llistxattr(cls, path, siz=0):
ret = cls._query_xattr(path, siz, 'llistxattr')
ret = cls._query_xattr(path.encode(), siz, 'llistxattr')
if isinstance(ret, str):
ret = ret.strip('\0')
ret = ret.split('\0') if ret else []
@ -73,13 +78,13 @@ class Xattr(object):
@classmethod
def lsetxattr(cls, path, attr, val):
ret = cls.libc.lsetxattr(path, attr, val, len(val), 0)
ret = cls.libc.lsetxattr(path.encode(), attr.encode(), val, len(val), 0)
if ret == -1:
cls.raise_oserr()
@classmethod
def lremovexattr(cls, path, attr):
ret = cls.libc.lremovexattr(path, attr)
ret = cls.libc.lremovexattr(path.encode(), attr.encode())
if ret == -1:
cls.raise_oserr()

18
geo-replication/syncdaemon/libgfchangelog.py
View File

@ -39,8 +39,8 @@ class Changes(object):
@classmethod
def cl_register(cls, brick, path, log_file, log_level, retries=0):
ret = cls._get_api('gf_changelog_register')(brick, path,
log_file,
ret = cls._get_api('gf_changelog_register')(brick.encode(), path.encode(),
log_file.encode(),
log_level, retries)
if ret == -1:
cls.raise_changelog_err()
@ -63,14 +63,14 @@ class Changes(object):
def clsort(f):
return f.split('.')[-1]
changes = []
buf = create_string_buffer('\0', 4096)
buf = create_string_buffer(b'\0' * 4096)
call = cls._get_api('gf_changelog_next_change')
while True:
ret = call(buf, 4096)
if ret in (0, -1):
break
changes.append(buf.raw[:ret - 1])
changes.append(buf.raw[:ret - 1].decode())
if ret == -1:
cls.raise_changelog_err()
# cleanup tracker
@ -79,7 +79,7 @@ class Changes(object):
@classmethod
def cl_done(cls, clfile):
ret = cls._get_api('gf_changelog_done')(clfile)
ret = cls._get_api('gf_changelog_done')(clfile.encode())
if ret == -1:
cls.raise_changelog_err()
@ -94,7 +94,7 @@ class Changes(object):
@classmethod
def cl_history_changelog(cls, changelog_path, start, end, num_parallel):
actual_end = c_ulong()
ret = cls._get_api('gf_history_changelog')(changelog_path, start, end,
ret = cls._get_api('gf_history_changelog')(changelog_path.encode(), start, end,
num_parallel,
byref(actual_end))
if ret == -1:
@ -118,14 +118,14 @@ class Changes(object):
return f.split('.')[-1]
changes = []
buf = create_string_buffer('\0', 4096)
buf = create_string_buffer(b'\0' * 4096)
call = cls._get_api('gf_history_changelog_next_change')
while True:
ret = call(buf, 4096)
if ret in (0, -1):
break
changes.append(buf.raw[:ret - 1])
changes.append(buf.raw[:ret - 1].decode())
if ret == -1:
cls.raise_changelog_err()
@ -133,6 +133,6 @@ class Changes(object):
@classmethod
def cl_history_done(cls, clfile):
ret = cls._get_api('gf_history_changelog_done')(clfile)
ret = cls._get_api('gf_history_changelog_done')(clfile.encode())
if ret == -1:
cls.raise_changelog_err()

4
geo-replication/syncdaemon/monitor.py
View File

@ -26,7 +26,9 @@ from syncdutils import set_term_handler, GsyncdError
from syncdutils import Thread, finalize, Volinfo, VolinfoFromGconf
from syncdutils import gf_event, EVENT_GEOREP_FAULTY, get_up_nodes
from gsyncdstatus import GeorepStatus, set_monitor_status
from syncdutils import unshare_propagation_supported, pipe
from syncdutils import unshare_propagation_supported
import py2py3
from py2py3 import pipe
ParseError = XET.ParseError if hasattr(XET, 'ParseError') else SyntaxError

49
geo-replication/syncdaemon/py2py3.py Normal file
View File

@ -0,0 +1,49 @@
#
# Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
# This file is part of GlusterFS.
# This file is licensed to you under your choice of the GNU Lesser
# General Public License, version 3 or any later version (LGPLv3 or
# later), or the GNU General Public License, version 2 (GPLv2), in all
# cases as published by the Free Software Foundation.
#
# All python2/python3 compatibility routines
import sys
import os
if sys.version_info >= (3,):
def pipe():
(r, w) = os.pipe()
os.set_inheritable(r, True)
os.set_inheritable(w, True)
return (r, w)
# Raw conversion of bytearray to string. Used in the cases where
# buffer is created by create_string_buffer which is a 8-bit char
# array and passed to syscalls to fetch results. Using encode/decode
# doesn't work as it converts to string altering the size.
def bytearray_to_str(byte_arr):
return ''.join([chr(b) for b in byte_arr])
# Raw conversion of string to bytes. This is required to convert
# back the string into bytearray(c char array) to use in struc
# pack/unpacking. Again encode/decode can't be used as it
# converts it alters size.
def str_to_bytearray(string):
return bytes([ord(c) for c in string])
else:
def pipe():
(r, w) = os.pipe()
return (r, w)
# Raw conversion of bytearray to string
def bytearray_to_str(byte_arr):
return ''.join([b for b in byte_arr])
# Raw conversion of string to bytearray
def str_to_bytearray(string):
return b"".join([c for c in string])

21
geo-replication/syncdaemon/resource.py
View File

@ -40,7 +40,9 @@ from syncdutils import GX_GFID_CANONICAL_LEN
from gsyncdstatus import GeorepStatus
from syncdutils import lf, Popen, sup
from syncdutils import Xattr, matching_disk_gfid, get_gfid_from_mnt
from syncdutils import unshare_propagation_supported, get_slv_dir_path, pipe
from syncdutils import unshare_propagation_supported, get_slv_dir_path
import py2py3
from py2py3 import pipe, str_to_bytearray
ENOTSUP = getattr(errno, 'ENOTSUP', 'EOPNOTSUPP')
@ -145,6 +147,7 @@ class Server(object):
if buf == ENOENT:
return buf
else:
buf = str_to_bytearray(buf)
m = re.match('(.{8})(.{4})(.{4})(.{4})(.{12})', "".join(
['%02x' % x for x in struct.unpack(cls.GFID_FMTSTR, buf)]))
return '-'.join(m.groups())
@ -235,6 +238,7 @@ class Server(object):
val = Xattr.lgetxattr(path,
'.'.join([cls.GX_NSPACE, uuid, 'xtime']),
8)
val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@ -257,6 +261,7 @@ class Server(object):
val = Xattr.lgetxattr(path,
'.'.join([cls.GX_NSPACE, uuid, 'stime']),
8)
val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@ -279,6 +284,7 @@ class Server(object):
val = Xattr.lgetxattr(path,
'.'.join([cls.GX_NSPACE, uuid, 'stime']),
8)
val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@ -302,6 +308,7 @@ class Server(object):
'.'.join([cls.GX_NSPACE, uuid,
'entry_stime']),
8)
val = str_to_bytearray(val)
return struct.unpack('!II', val)
except OSError:
ex = sys.exc_info()[1]
@ -375,7 +382,7 @@ class Server(object):
def entry_pack_reg(gf, bn, mo, uid, gid):
blen = len(bn)
return struct.pack(cls._fmt_mknod(blen),
uid, gid, gf, mo, bn,
uid, gid, gf.encode(), mo, bn.encode(),
stat.S_IMODE(mo), 0, umask())
def entry_pack_reg_stat(gf, bn, st):
@ -383,14 +390,14 @@ class Server(object):
mo = st['mode']
return struct.pack(cls._fmt_mknod(blen),
st['uid'], st['gid'],
gf, mo, bn,
gf.encode(), mo, bn.encode(),
stat.S_IMODE(mo), 0, umask())
# mkdir
def entry_pack_mkdir(gf, bn, mo, uid, gid):
blen = len(bn)
return struct.pack(cls._fmt_mkdir(blen),
uid, gid, gf, mo, bn,
uid, gid, gf.encode(), mo, bn.encode(),
stat.S_IMODE(mo), umask())
# symlink
@ -399,7 +406,8 @@ class Server(object):
llen = len(lnk)
return struct.pack(cls._fmt_symlink(blen, llen),
st['uid'], st['gid'],
gf, st['mode'], bn, lnk)
gf.encode(), st['mode'], bn.encode(),
lnk.encode())
def entry_purge(op, entry, gfid, e):
# This is an extremely racy code and needs to be fixed ASAP.
@ -450,7 +458,7 @@ class Server(object):
else:
en = e['entry']
disk_gfid = get_gfid_from_mnt(en)
if isinstance(disk_gfid, basestring) and \
if isinstance(disk_gfid, str) and \
e['gfid'] != disk_gfid:
slv_entry_info['gfid_mismatch'] = True
st = lstat(en)
@ -1022,6 +1030,7 @@ class GLUSTERServer(Server):
"""generic volume mark fetching/parsing backed"""
fmt_string = cls.NTV_FMTSTR + extra_fields
buf = Xattr.lgetxattr('.', xattr, struct.calcsize(fmt_string))
buf = str_to_bytearray(buf)
vm = struct.unpack(fmt_string, buf)
m = re.match(
'(.{8})(.{4})(.{4})(.{4})(.{12})',

12
geo-replication/syncdaemon/syncdutils.py
View File

@ -1021,15 +1021,3 @@ def get_up_nodes(hosts, port):
up_nodes.append(h)
return up_nodes
def pipe():
# Pipe routine for python2 and python3 compatiability
try:
(r, w) = os.pipe()
os.set_inheritable(r, True)
os.set_inheritable(w, True)
except AttributeError:
(r, w) = os.pipe()
return (r, w)