diff --git a/README.rst b/README.rst index 7d297dea..7a8e2a1b 100644 --- a/README.rst +++ b/README.rst @@ -46,6 +46,14 @@ Read and deserialize rosbag2 messages: print(msg.header.frame_id) +Convert rosbag1 to rosbag2:: + + # Convert "foo.bag", result will be "foo/" + rosbags-convert foo.bag + + # Convert "foo.bag", save the result as "bar" + rosbags-convert foo.bag --dst /path/to/bar + Documentation ============= diff --git a/docs/api/rosbags.convert.rst b/docs/api/rosbags.convert.rst new file mode 100644 index 00000000..ae336cf1 --- /dev/null +++ b/docs/api/rosbags.convert.rst @@ -0,0 +1,6 @@ +rosbags.convert +=============== + +.. automodule:: rosbags.convert + :members: + :show-inheritance: diff --git a/docs/api/rosbags.rst b/docs/api/rosbags.rst index 2dca8d05..19c8a4f3 100644 --- a/docs/api/rosbags.rst +++ b/docs/api/rosbags.rst @@ -4,6 +4,7 @@ Rosbags namespace .. toctree:: :maxdepth: 4 + rosbags.convert rosbags.rosbag1 rosbags.rosbag2 rosbags.serde diff --git a/docs/index.rst b/docs/index.rst index 583a285e..904b22ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ topics/serde topics/rosbag2 topics/rosbag1 + topics/convert .. toctree:: diff --git a/docs/topics/convert.rst b/docs/topics/convert.rst new file mode 100644 index 00000000..c4d2014d --- /dev/null +++ b/docs/topics/convert.rst @@ -0,0 +1,29 @@ +Convert Rosbag1 to Rosbag2 +========================== + +The :py:mod:`rosbags.convert` package includes a CLI tool to convert legacy rosbag1 files to rosbag2. + +Features +-------- + +- Reasonably fast, as it converts raw ROS1 messages to raw CDR messages without going though deserialization and serialization +- Tries to match ROS1 message type names to registered ROS2 types +- Automatically registers unknown message types present in the legacy rosbag file for the conversion +- Handles differences of ``std_msgs/msg/Header`` between both ROS versions + +Limitations +----------- + +- Refuses to convert unindexed rosbag files, please reindex files before conversion +- Currently does not handle split bags + +Usage +----- + +.. code-block:: console + + # Convert "foo.bag", result will be "foo/" + $ rosbags-convert foo.bag + + # Convert "foo.bag", save the result as "bar" + $ rosbags-convert foo.bag --dst /path/to/bar diff --git a/setup.cfg b/setup.cfg index 7b70f9fd..21c5151c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,6 +54,10 @@ install_requires = ruamel.yaml zstandard +[options.entry_points] +console_scripts = + rosbags-convert = rosbags.convert.__main__:main + [options.extras_require] dev = darglint diff --git a/src/rosbags/convert/__init__.py b/src/rosbags/convert/__init__.py new file mode 100644 index 00000000..e4eddce0 --- /dev/null +++ b/src/rosbags/convert/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020-2021 Ternaris. +# SPDX-License-Identifier: Apache-2.0 +"""Rosbags file format conversion. + +Conversion function transforms files from rosbag1 format to the latest rosbag2 +format. It automatically matches ROS1 message types to their ROS2 counterparts +and adds custom types not present in the type system. + +""" + +from .converter import ConverterError, convert + +__all__ = [ + 'ConverterError', + 'convert', +] diff --git a/src/rosbags/convert/__main__.py b/src/rosbags/convert/__main__.py new file mode 100644 index 00000000..2ab89904 --- /dev/null +++ b/src/rosbags/convert/__main__.py @@ -0,0 +1,62 @@ +# Copyright 2020-2021 Ternaris. +# SPDX-License-Identifier: Apache-2.0 +"""CLI tool for rosbag conversion.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +from .converter import ConverterError, convert + +if TYPE_CHECKING: + from typing import Callable + + +def pathtype(exists: bool = True) -> Callable: + """Path argument for argparse. + + Args: + exists: Path should exists in filesystem. + + Returns: + Argparse type function. + + """ + + def topath(pathname: str) -> Path: + path = Path(pathname) + if exists != path.exists(): + raise argparse.ArgumentTypeError( + f'{path} should {"exist" if exists else "not exist"}.', + ) + return path + + return topath + + +def main() -> None: + """Parse cli arguments and run conversion.""" + parser = argparse.ArgumentParser(description='Convert rosbag1 to rosbag2.') + parser.add_argument( + 'src', + type=pathtype(), + help='source path to read rosbag1 from', + ) + parser.add_argument( + '--dst', + type=pathtype(exists=False), + help='destination path for rosbag2', + ) + args = parser.parse_args() + try: + convert(args.src, args.dst) + except ConverterError as err: + print(f'ERROR: {err}') # noqa: T001 + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/src/rosbags/convert/converter.py b/src/rosbags/convert/converter.py new file mode 100644 index 00000000..f1e4e9f3 --- /dev/null +++ b/src/rosbags/convert/converter.py @@ -0,0 +1,55 @@ +# Copyright 2020-2021 Ternaris. +# SPDX-License-Identifier: Apache-2.0 +"""Rosbag1 to Rosbag2 Converter.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from rosbags.rosbag1 import Reader, ReaderError +from rosbags.rosbag2 import Writer, WriterError +from rosbags.serde import ros1_to_cdr +from rosbags.typesys import get_types_from_msg, register_types + +if TYPE_CHECKING: + from pathlib import Path + from typing import Any, Dict, Optional + + +class ConverterError(Exception): + """Converter Error.""" + + +def convert(src: Path, dst: Optional[Path]) -> None: + """Convert Rosbag1 to Rosbag2. + + Args: + src: Rosbag1 path. + dst: Rosbag2 path. + + Raises: + ConverterError: An error occured during reading, writing, or + converting. + + """ + dst = dst if dst else src.with_suffix('') + if dst.exists(): + raise ConverterError(f'Output path {str(dst)!r} exists already.') + + try: + with Reader(src) as reader, Writer(dst) as writer: + typs: Dict[str, Any] = {} + for name, topic in reader.topics.items(): + writer.add_topic(name, topic.msgtype) + typs.update(get_types_from_msg(topic.msgdef, topic.msgtype)) + register_types(typs) + + for topic, msgtype, timestamp, data in reader.messages(): + data = ros1_to_cdr(data, msgtype) + writer.write(topic, timestamp, data) + except ReaderError as err: + raise ConverterError(f'Reading source bag: {err}') from err + except WriterError as err: + raise ConverterError(f'Writing destination bag: {err}') from err + except Exception as err: # pylint: disable=broad-except + raise ConverterError(f'Converting rosbag: {err!r}') from err diff --git a/tests/test_convert.py b/tests/test_convert.py new file mode 100644 index 00000000..1d015f35 --- /dev/null +++ b/tests/test_convert.py @@ -0,0 +1,109 @@ +# Copyright 2020-2021 Ternaris. +# SPDX-License-Identifier: Apache-2.0 +"""Rosbag1to2 converter tests.""" + +import sys +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from rosbags.convert import ConverterError, convert +from rosbags.convert.__main__ import main +from rosbags.rosbag1 import ReaderError +from rosbags.rosbag2 import WriterError + + +def test_cliwrapper(tmp_path: Path): + """Test cli wrapper.""" + (tmp_path / 'subdir').mkdir() + (tmp_path / 'ros1.bag').write_text('') + + with patch('rosbags.convert.__main__.convert') as cvrt, \ + patch.object(sys, 'argv', ['cvt']), \ + pytest.raises(SystemExit): + main() + assert not cvrt.called + + with patch('rosbags.convert.__main__.convert') as cvrt, \ + patch.object(sys, 'argv', ['cvt', str(tmp_path / 'no.bag')]), \ + pytest.raises(SystemExit): + main() + assert not cvrt.called + + with patch('rosbags.convert.__main__.convert') as cvrt, \ + patch.object(sys, 'argv', ['cvt', str(tmp_path / 'ros1.bag')]): + main() + cvrt.assert_called_with(tmp_path / 'ros1.bag', None) + + with patch('rosbags.convert.__main__.convert') as cvrt, \ + patch.object(sys, 'argv', ['cvt', + str(tmp_path / 'ros1.bag'), + '--dst', + str(tmp_path / 'subdir')]), \ + pytest.raises(SystemExit): + main() + assert not cvrt.called + + with patch('rosbags.convert.__main__.convert') as cvrt, \ + patch.object(sys, 'argv', ['cvt', + str(tmp_path / 'ros1.bag'), + '--dst', + str(tmp_path / 'target')]): + main() + cvrt.assert_called_with(tmp_path / 'ros1.bag', tmp_path / 'target') + + with patch.object(sys, 'argv', ['cvt', str(tmp_path / 'ros1.bag')]), \ + patch('builtins.print') as mock_print, \ + patch('rosbags.convert.__main__.convert', side_effect=ConverterError('exc')), \ + pytest.raises(SystemExit): + main() + mock_print.assert_called_with('ERROR: exc') + + +def test_convert(tmp_path: Path): + """Test conversion function.""" + (tmp_path / 'subdir').mkdir() + (tmp_path / 'foo.bag').write_text('') + + with pytest.raises(ConverterError, match='exists already'): + convert(Path('foo.bag'), tmp_path / 'subdir') + + with patch('rosbags.convert.converter.Reader') as reader, \ + patch('rosbags.convert.converter.Writer') as writer, \ + patch('rosbags.convert.converter.get_types_from_msg', return_value={'typ': 'def'}), \ + patch('rosbags.convert.converter.register_types') as register_types, \ + patch('rosbags.convert.converter.ros1_to_cdr') as ros1_to_cdr: + + reader.return_value.__enter__.return_value.topics = { + '/topic': Mock(msgtype='typ', msgdef='def'), + } + reader.return_value.__enter__.return_value.messages.return_value = [ + ('/topic', 'typ', 42, b'\x42'), + ] + + ros1_to_cdr.return_value = b'666' + + convert(Path('foo.bag'), None) + + reader.assert_called_with(Path('foo.bag')) + reader.return_value.__enter__.return_value.messages.assert_called_with() + + writer.assert_called_with(Path('foo')) + writer.return_value.__enter__.return_value.add_topic.assert_called_with('/topic', 'typ') + writer.return_value.__enter__.return_value.write.assert_called_with('/topic', 42, b'666') + + register_types.assert_called_with({'typ': 'def'}) + ros1_to_cdr.assert_called_with(b'\x42', 'typ') + + ros1_to_cdr.side_effect = KeyError('exc') + with pytest.raises(ConverterError, match='Converting rosbag: '): + convert(Path('foo.bag'), None) + + writer.side_effect = WriterError('exc') + with pytest.raises(ConverterError, match='Writing destination bag: '): + convert(Path('foo.bag'), None) + + reader.side_effect = ReaderError('exc') + with pytest.raises(ConverterError, match='Reading source bag: '): + convert(Path('foo.bag'), None)